Hello community,
here is the log from the commit of package clicfs for openSUSE:Factory
checked in at Tue May 5 18:04:50 CEST 2009.
--------
--- clicfs/clicfs.changes 2009-04-20 21:36:23.000000000 +0200
+++ /mounts/work_src_done/STABLE/clicfs/clicfs.changes 2009-05-05 14:13:52.000000000 +0200
@@ -1,0 +2,5 @@
+Tue May 5 14:13:39 CEST 2009 - coolo@suse.de
+
+- if the cow file is read only, then just read it
+
+-------------------------------------------------------------------
calling whatdependson for head-i586
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ clicfs.spec ++++++
--- /var/tmp/diff_new_pack.Q15012/_old 2009-05-05 18:04:41.000000000 +0200
+++ /var/tmp/diff_new_pack.Q15012/_new 2009-05-05 18:04:41.000000000 +0200
@@ -23,7 +23,7 @@
Requires: fuse
Summary: Compressed Loop Image Container
Version: 1.0
-Release: 3
+Release: 4
License: GPL v2 only
Group: System/Filesystems
Source: clicfs.tar.bz2
@@ -62,6 +62,8 @@
%_mandir/man1/*
%changelog
+* Tue May 05 2009 coolo@suse.de
+- if the cow file is read only, then just read it
* Mon Apr 20 2009 coolo@suse.de
- fixing license
* Mon Apr 20 2009 coolo@suse.de
++++++ clicfs.tar.bz2 ++++++
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/doc/clicfs.1 new/doc/clicfs.1
--- old/doc/clicfs.1 2009-04-20 20:31:33.000000000 +0200
+++ new/doc/clicfs.1 2009-05-05 14:13:03.000000000 +0200
@@ -182,7 +182,7 @@
The compressed content is actually read only (as with most compressed file systems), but if a write happens, the data is either stored in memory (with \-m) or written into a cowfile (with \-c)\&. The container is seen in the mount point as actually larger than the original file, so it can be resizes to match the memory available\&.
.SH "See Also"
.PP
-mkclicfs, undoenerfs
+mkclicfs, unclicfs
.SH "Author"
.PP
Stephan Kulow
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/doc/man-mkclicfs.1.docbook new/doc/man-mkclicfs.1.docbook
--- old/doc/man-mkclicfs.1.docbook 2009-04-20 20:31:33.000000000 +0200
+++ new/doc/man-mkclicfs.1.docbook 2009-05-05 14:13:03.000000000 +0200
@@ -44,6 +44,9 @@
<arg choice="opt">
<option>-l</option> <arg choice="plain">logfile</arg>
</arg>
+<arg choice="opt">
+ <option>-n</option> <arg choice="plain">cores</arg>
+</arg>
<arg choice="plain">infile</arg>
<arg choice="plain">outfile</arg>
</cmdsynopsis>
@@ -56,6 +59,7 @@
<para>The blocksize defines how many pages are grouped together before compression. The default is 32, it is a good balance between overhead and compression ratio.</para>
<para>The compression preset defines on how slow it is or how good it compresses. Compare with xz presets. -c 0 is fast and -c 9 is (very) slow - but the compression rate varies a lot - of course depending on the data. Preset 6 is the default.</para>
<para>The logfile is the logfile that is generated from clicfs -l, clicfs will output the access pattern and mkclicfs will reorder the blocks so they are in order.</para>
+<para>The number of cores is by default taken from the system. Sometimes it's wise to not all of them though.</para>
</refsect1>
<refsect1>
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/doc/mkclicfs.1 new/doc/mkclicfs.1
--- old/doc/mkclicfs.1 2009-04-20 20:31:33.000000000 +0200
+++ new/doc/mkclicfs.1 2009-05-05 14:13:03.000000000 +0200
@@ -171,18 +171,24 @@
.SH "Synopsis"
.fam C
.HP \w'\fBmkclicfs\fR\ 'u
-\fBmkclicfs\fR [\fB\-d\fR] [\fB\-p\fR\ pagesize] [\fB\-b\fR\ blocksize] [\fB\-c\fR\ compression] [\fB\-l\fR\ logfile] infile outfile
+\fBmkclicfs\fR [\fB\-d\fR] [\fB\-p\fR\ pagesize] [\fB\-b\fR\ blocksize] [\fB\-c\fR\ compression] [\fB\-l\fR\ logfile] [\fB\-n\fR\ cores] infile outfile
.fam
.SH "Description"
.PP
-The infile is a loop image (so far only tested with ext3) that is assumed to use 4K pages\&. The outfile is the Compressed Loop Image Container (Clic), it will contain X parts with blocksize\&. mkclicfs automatically discards duplicated pages\&. A good value for blocksize is 131072, it is a good balance between overhead and compression ratio and it will be default in future releases\&.
+The infile is a loop image (so far only tested with ext3) that is assumed to use pages of pagesize\&. The outfile is the Compressed Loop Image Container (Clic), it will contain X parts with blocksize\&. mkclicfs automatically discards duplicated pages, unless \-d is given\&.
.PP
-The preset defines on how slow it is or how good it compresses\&. Compare with xz presets\&. \-0 is fast and \-9 is (very) slow \- but the compression rate varies a lot \- of course depending on the data\&. Preset 6 is the default\&.
+The default for the pagesize is 4096, the default for ext3
.PP
-The profile is the logfile that is generated from clicfs \-l, clicfs will output the access pattern and mkclicfs will reorder the blocks so they are in order\&.
+The blocksize defines how many pages are grouped together before compression\&. The default is 32, it is a good balance between overhead and compression ratio\&.
+.PP
+The compression preset defines on how slow it is or how good it compresses\&. Compare with xz presets\&. \-c 0 is fast and \-c 9 is (very) slow \- but the compression rate varies a lot \- of course depending on the data\&. Preset 6 is the default\&.
+.PP
+The logfile is the logfile that is generated from clicfs \-l, clicfs will output the access pattern and mkclicfs will reorder the blocks so they are in order\&.
+.PP
+The number of cores is by default taken from the system\&. Sometimes it\'s wise to not all of them though\&.
.SH "See Also"
.PP
-clicfs, undoenerfs
+clicfs, unclicfs
.SH "Author"
.PP
Stephan Kulow
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/doc/unclicfs.1 new/doc/unclicfs.1
--- old/doc/unclicfs.1 2009-04-20 20:31:33.000000000 +0200
+++ new/doc/unclicfs.1 2009-05-05 14:13:03.000000000 +0200
@@ -178,7 +178,7 @@
Unclicfs will extract the original file into the current directory\&. It\'s name was given to mkclicfs as input file\&.
.SH "See Also"
.PP
-mkclicfs, undoenerfs
+mkclicfs, clicfs
.SH "Author"
.PP
Stephan Kulow
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/src/clicfs.c new/src/clicfs.c
--- old/src/clicfs.c 2009-04-20 20:31:33.000000000 +0200
+++ new/src/clicfs.c 2009-05-05 14:13:03.000000000 +0200
@@ -50,7 +50,7 @@
static int clic_write_cow()
{
- if (!cowfilename)
+ if (!cowfilename || cowfile_ro == 1)
return 0;
uint32_t indexlen = sizeof(uint32_t) * 2;
@@ -530,8 +530,12 @@
free(packfilename);
if (cowfilename) {
- if (access(cowfilename, W_OK)) {
+ if (access(cowfilename, R_OK)) {
FILE *cow = fopen(cowfilename, "w");
+ if (!cow) {
+ perror("opening cow");
+ return 1;
+ }
uint32_t stringlen = (thefilesize / pagesize * pagesize) + 512 * 1024 * 1024;
fwrite((char*)&stringlen, 1, sizeof(uint32_t), cow);
stringlen = 0;
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/src/clicfs_common.c new/src/clicfs_common.c
--- old/src/clicfs_common.c 2009-04-20 20:31:33.000000000 +0200
+++ new/src/clicfs_common.c 2009-05-05 14:13:03.000000000 +0200
@@ -44,6 +44,7 @@
uint32_t cow_pages = 0;
uint32_t *cows = 0;
unsigned int cows_index = 0;
+int cowfile_ro = 0;
static lzma_stream strm;
@@ -68,10 +69,18 @@
int clicfs_read_cow(const char *cowfilename)
{
cowfilefd = open(cowfilename, O_RDWR);
+
if (cowfilefd == -1 ) {
- fprintf(stderr, "cowfile %s can't be opened\n", cowfilename);
- return 1;
- }
+
+ cowfilefd = open(cowfilename, O_RDONLY);
+ if (cowfilefd == -1) {
+ fprintf(stderr, "cowfile %s can't be opened\n", cowfilename);
+ return 1;
+ } else {
+ cowfile_ro = 1;
+ }
+ } else
+ cowfile_ro = 0;
struct stat st;
fstat(cowfilefd, &st);
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/src/clicfs.h new/src/clicfs.h
--- old/src/clicfs.h 2009-04-20 20:31:33.000000000 +0200
+++ new/src/clicfs.h 2009-05-05 14:13:03.000000000 +0200
@@ -39,6 +39,8 @@
extern uint32_t num_pages;
// the number of pages in the cow index
extern uint32_t cow_pages;
+// support temporary changes on ro medium
+extern int cowfile_ro;
#define DOENER_COW_COUNT 100
diff -urN --exclude=CVS --exclude=.cvsignore --exclude=.svn --exclude=.svnignore old/src/mkclicfs.cpp new/src/mkclicfs.cpp
--- old/src/mkclicfs.cpp 2009-04-20 20:31:33.000000000 +0200
+++ new/src/mkclicfs.cpp 2009-05-05 14:13:03.000000000 +0200
@@ -30,6 +30,11 @@
#include
#include
#include
+#include
+#include
+#include
+#include
+#include
#include
#include <map>
@@ -85,17 +90,345 @@
return strm.total_out;
}
+int blocksize = 32;
+int infd = -1;
+uint32_t *ublocks = 0;
+uint32_t *found = 0;
+bool check_dups = true;
+uint32_t *blockindex = 0;
+uint32_t num_pages = 0;
+size_t pagesize = 4096;
+uint32_t pindex = 0;
+int preset = 2;
+
+/* most of this logic is from mksquashfs - GPLv2+ */
+
+/* struct describing queues used to pass data between threads */
+struct queue {
+ int size;
+ int readp;
+ int writep;
+ pthread_mutex_t mutex;
+ pthread_cond_t empty;
+ pthread_cond_t full;
+ void **data;
+};
+
+static struct queue *from_reader, *to_writer;
+static pthread_t *thread;
+
+static int processors = -1;
+
+struct queue *queue_init(int size)
+{
+ struct queue *queue = (struct queue*)malloc(sizeof(struct queue));
+
+ if(queue == NULL)
+ return NULL;
+
+ if((queue->data = (void**)malloc(sizeof(void *) * (size + 1))) == NULL) {
+ free(queue);
+ return NULL;
+ }
+
+ queue->size = size + 1;
+ queue->readp = queue->writep = 0;
+ pthread_mutex_init(&queue->mutex, NULL);
+ pthread_cond_init(&queue->empty, NULL);
+ pthread_cond_init(&queue->full, NULL);
+
+ return queue;
+}
+
+int queue_length(struct queue *queue)
+{
+ pthread_mutex_lock(&queue->mutex);
+ int ret = (queue->writep - queue->readp + queue->size) % queue->size;
+ pthread_mutex_unlock(&queue->mutex);
+ return ret;
+}
+
+void queue_put(struct queue *queue, void *data)
+{
+ int nextp;
+
+ pthread_mutex_lock(&queue->mutex);
+
+ while((nextp = (queue->writep + 1) % queue->size) == queue->readp)
+ pthread_cond_wait(&queue->full, &queue->mutex);
+
+ queue->data[queue->writep] = data;
+ queue->writep = nextp;
+ pthread_cond_signal(&queue->empty);
+ pthread_mutex_unlock(&queue->mutex);
+}
+
+
+void *queue_get(struct queue *queue)
+{
+ void *data;
+ pthread_mutex_lock(&queue->mutex);
+
+ while(queue->readp == queue->writep)
+ pthread_cond_wait(&queue->empty, &queue->mutex);
+
+ data = queue->data[queue->readp];
+ queue->readp = (queue->readp + 1) % queue->size;
+ pthread_cond_signal(&queue->full);
+ pthread_mutex_unlock(&queue->mutex);
+
+ return data;
+}
+
+// the number of really saved parts
+uint32_t parts = 0;
+
+struct inbuf_struct {
+ size_t readin, totalin;
+ unsigned char *inbuf;
+ size_t part, bpp;
+ bool lastblock;
+};
+
+
+void *reader(void *arg)
+{
+ int oldstate;
+
+ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
+ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate);
+
+ uint32_t rindex = 0; // overall index
+ uint32_t uindex = 0; // index for "unused" blocks
+
+ std::mapstd::string,uint32_t dups;
+
+ uint32_t currentblocksperpart = 0; // for debug output
+
+ uint32_t usedblock = 0; // overall "mapped" index
+
+ while ( rindex < num_pages ) {
+
+ int currentblocks = 0;
+
+ inbuf_struct *in = new inbuf_struct();
+ in->inbuf = new unsigned char[blocksize*pagesize];
+ in->readin = 0;
+ in->totalin = 0;
+ in->lastblock = false;
+
+ while ( currentblocks < blocksize ) {
+ off_t cindex = 0;
+ if (rindex < pindex) {
+ cindex = ublocks[rindex];
+ } else {
+ while (found[uindex] && uindex < num_pages) uindex++;
+ assert( uindex < num_pages );
+ if ( uindex < num_pages ) {
+ cindex = uindex;
+ uindex++;
+ }
+ }
+ if ( lseek( infd, cindex * pagesize, SEEK_SET) == -1 ) {
+ perror( "seek" ); exit( 1 );
+ }
+ size_t diff= read( infd, in->inbuf + in->readin, pagesize);
+ in->totalin += diff;
+ std::string sm;
+ if (check_dups)
+ sm = calc_md5( in->inbuf + in->readin, diff );
+ //fprintf( stderr, "block %ld %s\n", ( long )cindex, sm.c_str() );
+ if ( check_dups && dups.find( sm ) != dups.end() ) {
+ //fprintf( stderr, "already have %s\n", sm.c_str() );
+ blockindex[cindex] = dups[sm];
+ } else {
+ blockindex[cindex] = usedblock++;
+ dups[sm] = blockindex[cindex];
+ in->readin += diff;
+ currentblocks++;
+ }
+ //fprintf(stderr, "block %ld in part %ld\n", cindex, parts);
+ rindex++;
+ currentblocksperpart++;
+ if ( rindex == num_pages ) {
+ in->lastblock = true;
+ break;
+ }
+ }
+ in->part = parts++;
+ in->bpp = currentblocksperpart;
+ currentblocksperpart = 0;
+ //fprintf( stderr, "put part %d %d\n", in->part, queue_length(from_reader));
+ queue_put( from_reader, in );
+
+ }
+ fprintf( stderr, "thread 0 is gone\n" );
+
+ thread[0] = 0;
+
+ pthread_exit(NULL);
+}
+
+
+struct outbuf_struct
+{
+ unsigned char *outbuf;
+ size_t insize, outsize, totalin;
+ size_t part, bpp;
+ bool lastblock;
+
+ ~outbuf_struct() {
+ delete [] outbuf;
+ }
+};
+
+void *deflator(void *arg)
+{
+ int oldstate;
+
+ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
+ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate);
+
+ while(1) {
+ inbuf_struct *in = (inbuf_struct*)queue_get(from_reader);
+
+ outbuf_struct *out = new outbuf_struct();
+ out->outbuf = new unsigned char[blocksize*pagesize + 300];
+// fprintf( stderr, "compress start %ld %d %x %ld\n", pthread_self(), in->part, in->inbuf, in->readin );
+ out->outsize = compress(preset, in->inbuf, in->readin, out->outbuf, blocksize*pagesize + 300);
+ out->part = in->part;
+ out->insize = in->readin;
+ out->bpp = in->bpp;
+ out->lastblock = in->lastblock;
+ out->totalin = in->totalin;
+
+// fprintf( stderr, "compress %ld %d %x %ld -> %ld\n", pthread_self(), in->part, in->inbuf, in->readin, out->outsize );
+ delete [] in->inbuf;
+ delete in;
+
+ queue_put(to_writer, out);
+ }
+
+ return 0;
+}
+
+void initialise_threads()
+{
+ int i;
+ sigset_t sigmask, old_mask;
+
+ sigemptyset(&sigmask);
+ sigaddset(&sigmask, SIGINT);
+ sigaddset(&sigmask, SIGQUIT);
+ if(sigprocmask(SIG_BLOCK, &sigmask, &old_mask) == -1)
+ exit( 1 );
+
+ if(processors == -1) {
+#ifndef linux
+ int mib[2];
+ size_t len = sizeof(processors);
+
+ mib[0] = CTL_HW;
+#ifdef HW_AVAILCPU
+ mib[1] = HW_AVAILCPU;
+#else
+ mib[1] = HW_NCPU;
+#endif
+
+ if(sysctl(mib, 2, &processors, &len, NULL, 0) == -1) {
+ ERROR("Failed to get number of available processors. Defaulting to 1\n");
+ processors = 1;
+ }
+#else
+ processors = get_nprocs();
+#endif
+ }
+
+ if((thread = (pthread_t*)malloc((2 + processors * 2) * sizeof(pthread_t))) == NULL)
+ exit( 1 );
+
+ from_reader = queue_init(20);
+ to_writer = queue_init(20);
+ pthread_create(&thread[0], NULL, reader, NULL);
+
+ for(i = 0; i < processors; i++) {
+ if(pthread_create(&thread[2+i], NULL, deflator, NULL) != 0 )
+ exit( 1 );
+ }
+
+ fprintf(stderr, "Parallel mkclicfs: Using %d processor%s\n", processors,
+ processors == 1 ? "" : "s");
+
+ if(sigprocmask(SIG_SETMASK, &old_mask, NULL) == -1)
+ exit( 1 );
+}
+
+int writer(size_t oparts, off_t index_off, FILE *out, uint64_t *sizes, uint64_t *offs, size_t full_size)
+{
+ int oldstate;
+
+ uint64_t total_in = 0;
+ uint64_t total_out = 0;
+
+ pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &oldstate);
+ pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &oldstate);
+
+ outbuf_struct **comps = new outbuf_struct*[oparts];
+ memset( comps, 0, sizeof( void* )*oparts );
+ size_t lastpart = -1;
+
+ int lastpercentage = 0;
+
+ struct timeval start;
+ gettimeofday(&start, 0);
+
+ while(1) {
+ outbuf_struct *comp = (outbuf_struct*)queue_get(to_writer);
+ //fprintf(stderr, "got %ld %ld %d %d\n", comp->part, lastpart, queue_length(to_writer), queue_length(from_reader));
+ comps[comp->part] = comp;
+
+ while ( comps[lastpart + 1] ) {
+ comp = comps[++lastpart];
+
+ sizes[comp->part] = comp->outsize;
+ offs[comp->part] = total_out + index_off;
+ total_in += comp->totalin;
+ total_out += comp->outsize;
+ if (fwrite(comp->outbuf, comp->outsize, 1, out) != 1) {
+ perror("write"); return 1;
+ }
+
+ if ((int)(100 * total_in / full_size) > lastpercentage || comp->lastblock ) {
+ struct timeval current;
+ gettimeofday(¤t, 0);
+ fprintf(stderr, "part blocks:%d%% parts:%ld total:%d%% time:%d\n",
+ lastpercentage+1, (long)comp->part,
+ (int)(total_out * 100 / total_in), (current.tv_sec - start.tv_sec) * 1000 + ((current.tv_usec - start.tv_usec) / 1000 ));
+ start.tv_sec = current.tv_sec;
+ start.tv_usec = current.tv_usec;
+ lastpercentage++;
+ }
+
+ if ( comp->lastblock ) {
+ delete comp;
+ goto out;
+ }
+ comps[comp->part] = 0;
+ delete comp;
+ }
+ }
+out:
+ delete [] comps;
+ return 0;
+}
+
int main(int argc, char **argv)
{
- bool check_dups = true;
- int blocksize = 32;
- int pagesize = 4096;
const char *profile = 0;
- int preset = 2;
bool usage = false;
int opt;
- while ((opt = getopt(argc, argv, "dp:b:l:c:")) != -1) {
+ while ((opt = getopt(argc, argv, "dp:b:l:c:n:")) != -1) {
switch (opt) {
case 'd':
check_dups = false;
@@ -118,6 +451,11 @@
if (preset < 0 || preset > 9)
usage = true;
break;
+ case 'n':
+ processors = atoi(optarg);
+ if (processors < 1)
+ usage = true;
+ break;
default: /* '?' */
usage = true;
break;
@@ -136,7 +474,7 @@
struct stat st;
stat(infile, &st);
- uint32_t num_pages = st.st_size / pagesize;
+ num_pages = st.st_size / pagesize;
/* ext3 should be X blocks */
if (num_pages * pagesize != st.st_size)
num_pages++;
@@ -146,12 +484,11 @@
if (oparts * blocksize != num_pages)
oparts++;
- uint32_t *found = ( uint32_t* )malloc(sizeof(uint32_t)*num_pages);
+ found = ( uint32_t* )malloc(sizeof(uint32_t)*num_pages);
memset(found, 0, sizeof(int)*num_pages);
- uint32_t *ublocks = ( uint32_t* )malloc(sizeof(uint32_t)*num_pages);
+ ublocks = ( uint32_t* )malloc(sizeof(uint32_t)*num_pages);
memset(ublocks, 0, sizeof(int)*num_pages);
- uint32_t pindex = 0;
// always take the first block to make the algorithm easier
ublocks[pindex++] = 0;
found[0] = pindex;
@@ -178,24 +515,17 @@
fprintf(stderr, "pindex %ld %ld\n", (long)pindex, (long)num_pages);
- int infd = open(infile, O_RDONLY);
+ infd = open(infile, O_RDONLY);
FILE *out = fopen(outfile, "w");
if (!out) {
perror("open output");
return 1;
}
- unsigned char inbuf[blocksize*pagesize];
- unsigned char outbuf[blocksize*pagesize + 300];
-
- uint64_t total_in = 0;
- uint64_t total_out = 0;
-
uint64_t *sizes = ( uint64_t* )malloc(sizeof(uint64_t)*oparts);
uint64_t *offs = ( uint64_t* )malloc(sizeof(uint64_t)*oparts);
off_t index_off = 6;
- int lastpercentage = 0;
assert( DOENER_MAGIC < 100 );
fprintf(out, "CLIC%02d", DOENER_MAGIC );
@@ -226,91 +556,22 @@
off_t index_blocks = index_off;
index_off += num_pages * sizeof( uint32_t );
- uint32_t *blockmap = new uint32_t[num_pages];
+ blockindex = new uint32_t[num_pages];
off_t index_part = index_off;
index_off += 2 * oparts * sizeof(uint64_t) + sizeof(uint32_t);
fseek(out, index_off, SEEK_SET);
- uint32_t rindex = 0; // overall index
- uint32_t uindex = 0; // index for "unused" blocks
-
- std::mapstd::string,uint32_t dups;
-
- // the number of really saved parts
- uint32_t parts = 0;
- uint32_t currentblocksperpart = 0; // for debug output
- uint32_t lastparts = 0; // for debug output
-
- uint32_t usedblock = 0; // overall "mapped" index
-
- while ( rindex < num_pages )
- {
- int currentblocks = 0;
- size_t readin = 0;
-
- while ( currentblocks < blocksize )
- {
- off_t cindex = 0;
- if (rindex < pindex) {
- cindex = ublocks[rindex];
- } else {
- while (found[uindex] && uindex < num_pages) uindex++;
- assert( uindex < num_pages );
- if ( uindex < num_pages ) {
- cindex = uindex;
- uindex++;
- }
- }
- if ( lseek( infd, cindex * pagesize, SEEK_SET) == -1 ) {
- perror( "seek" ); return 1;
- }
- size_t diff= read( infd, inbuf+readin, pagesize);
- std::string sm;
- if (check_dups)
- sm = calc_md5( inbuf+readin, diff );
- if ( check_dups && dups.find( sm ) != dups.end() ) {
- //fprintf( stderr, "already have %s\n", sm.c_str() );
- blockmap[cindex] = dups[sm];
- } else {
- blockmap[cindex] = usedblock++;
- dups[sm] = blockmap[cindex];
- readin += diff;
- currentblocks++;
- }
- //fprintf(stderr, "block %ld in part %ld\n", cindex, parts);
- rindex++;
- currentblocksperpart++;
- if ( rindex == num_pages )
- break;
- }
- size_t outsize = compress(preset, inbuf, readin, outbuf, blocksize*pagesize + 300);
- sizes[parts] = outsize;
- offs[parts] = total_out + index_off;
- total_in += readin;
- total_out += outsize;
- if (fwrite(outbuf, outsize, 1, out) != 1) {
- perror("write"); return 1;
- }
-
- parts++;
- if ((int)(rindex * 100. / num_pages) > lastpercentage || rindex >= num_pages - 1 && parts > lastparts && readin ) {
- fprintf(stderr, "part blocks:%d%% parts:%ld bpp:%d current:%d%% total:%d%%\n",
- (int)(rindex * 100. / num_pages), (long)parts,
- (int)( currentblocksperpart / ( parts - lastparts ) ),
- (int)(outsize * 100 / readin), (int)(total_out * 100 / total_in));
- lastpercentage++;
- lastparts = parts;
- currentblocksperpart = 0;
- }
- }
+ initialise_threads();
+ if ( writer(oparts, index_off, out, sizes, offs, st.st_size) )
+ return 1;
if (fseek(out, index_blocks, SEEK_SET) < 0) {
perror("seek"); return 1;
}
for (i = 0; i < num_pages; ++i)
- if (!writeindex(out, blockmap[i]))
+ if (!writeindex(out, blockindex[i]))
return 1;
if (fseek(out, index_part, SEEK_SET) < 0) {
@@ -330,7 +591,9 @@
fclose(out);
close( infd );
- delete [] blockmap;
+ delete [] blockindex;
+ free(offs);
+ free(sizes);
return 0;
}
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Remember to have fun...
--
To unsubscribe, e-mail: opensuse-commit+unsubscribe@opensuse.org
For additional commands, e-mail: opensuse-commit+help@opensuse.org