Hello community,
here is the log from the commit of package snappy for openSUSE:Factory checked in at 2013-11-15 13:39:00
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/snappy (Old)
and /work/SRC/openSUSE:Factory/.snappy.new (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "snappy"
Changes:
--------
--- /work/SRC/openSUSE:Factory/snappy/snappy.changes 2013-09-05 23:26:03.000000000 +0200
+++ /work/SRC/openSUSE:Factory/.snappy.new/snappy.changes 2013-11-15 13:39:01.000000000 +0100
@@ -1,0 +2,12 @@
+Wed Nov 13 12:44:42 UTC 2013 - p.drouand@gmail.com
+
+- Update to version 1.1.1
+ * Add support for uncompressing to iovecs (scatter I/O).
+ * Speed up decompression by ~2%; much more so (~13-20%) on
+ a few benchmarks on given compilers and CPUs.
+ * Fix a few issues with MSVC compilation.
+ * Support truncated test data in the benchmark.
+- Adapt patch to upstream changes
+ * snappy-random-return.patch > snappy-1.1.1-random-return.patch
+
+-------------------------------------------------------------------
Old:
----
snappy-1.1.0.tar.gz
snappy-random-return.patch
New:
----
snappy-1.1.1-random-return.patch
snappy-1.1.1.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ snappy.spec ++++++
--- /var/tmp/diff_new_pack.Omzn4A/_old 2013-11-15 13:39:03.000000000 +0100
+++ /var/tmp/diff_new_pack.Omzn4A/_new 2013-11-15 13:39:03.000000000 +0100
@@ -18,7 +18,7 @@
%define lib_name libsnappy1
Name: snappy
-Version: 1.1.0
+Version: 1.1.1
Release: 0
Summary: A fast compressor/decompressor library
License: BSD-3-Clause
@@ -26,7 +26,7 @@
Url: http://code.google.com/p/snappy/
Source0: http://snappy.googlecode.com/files/%{name}-%{version}.tar.gz
#PATCH-FIX-OPENSUSE: fix minor things in unittest, probably not worth for upstreaming ...
-Patch0: snappy-random-return.patch
+Patch0: snappy-1.1.1-random-return.patch
BuildRoot: %{_tmppath}/%{name}-%{version}-build
BuildRequires: gcc-c++
BuildRequires: lzo-devel
++++++ snappy-1.1.1-random-return.patch ++++++
---
snappy-test.h | 8 ++++++--
snappy_unittest.cc | 1 +
2 files changed, 7 insertions(+), 2 deletions(-)
Index: snappy_unittest.cc
===================================================================
--- snappy_unittest.cc.orig
+++ snappy_unittest.cc
@@ -161,6 +161,7 @@ static size_t MinimumRequiredOutputSpace
default:
LOG(FATAL) << "Unknown compression type number " << comp;
+ return -1; //make gcc happy
}
}
Index: snappy-test.h
===================================================================
--- snappy-test.h.orig
+++ snappy-test.h
@@ -132,13 +132,15 @@
} // namespace File
namespace file {
- int Defaults() { }
+ int Defaults() { return 0; } //make gcc happy
class DummyStatus {
public:
void CheckSuccess() { }
};
+ DummyStatus _dummy_ret;
+
DummyStatus GetContents(const string& filename, string* data, int unused) {
FILE* fp = fopen(filename.c_str(), "rb");
if (fp == NULL) {
@@ -158,6 +160,7 @@
}
fclose(fp);
+ return _dummy_ret; //make gcc happy
}
DummyStatus SetContents(const string& filename,
@@ -176,6 +179,7 @@
}
fclose(fp);
+ return _dummy_ret; //make gcc happy
}
} // namespace file
++++++ snappy-1.1.0.tar.gz -> snappy-1.1.1.tar.gz ++++++
++++ 1603 lines of diff (skipped)
++++ retrying with extended exclude list
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/ChangeLog new/snappy-1.1.1/ChangeLog
--- old/snappy-1.1.0/ChangeLog 2013-02-05 15:36:32.000000000 +0100
+++ new/snappy-1.1.1/ChangeLog 2013-10-15 12:41:47.000000000 +0200
@@ -1,4 +1,272 @@
------------------------------------------------------------------------
+r80 | snappy.mirrorbot@gmail.com | 2013-08-13 14:55:00 +0200 (Tue, 13 Aug 2013) | 6 lines
+
+Add autoconf tests for size_t and ssize_t. Sort-of resolves public issue 79;
+it would solve the problem if MSVC typically used autoconf. However, it gives
+a natural place (config.h) to put the typedef even for MSVC.
+
+R=jsbell
+
+------------------------------------------------------------------------
+r79 | snappy.mirrorbot@gmail.com | 2013-07-29 13:06:44 +0200 (Mon, 29 Jul 2013) | 14 lines
+
+When we compare the number of bytes produced with the offset for a
+backreference, make the signedness of the bytes produced clear,
+by sticking it into a size_t. This avoids a signed/unsigned compare
+warning from MSVC (public issue 71), and also is slightly clearer.
+
+Since the line is now so long the explanatory comment about the -1u
+trick has to go somewhere else anyway, I used the opportunity to
+explain it in slightly more detail.
+
+This is a purely stylistic change; the emitted assembler from GCC
+is identical.
+
+R=jeff
+
+------------------------------------------------------------------------
+r78 | snappy.mirrorbot@gmail.com | 2013-06-30 21:24:03 +0200 (Sun, 30 Jun 2013) | 111 lines
+
+In the fast path for decompressing literals, instead of checking
+whether there's 16 bytes free and then checking right afterwards
+(when having subtracted the literal size) that there are now
+5 bytes free, just check once for 21 bytes. This skips a compare
+and a branch; although it is easily predictable, it is still
+a few cycles on a fast path that we would like to get rid of.
+
+Benchmarking this yields very confusing results. On open-source
+GCC 4.8.1 on Haswell, we get exactly the expected results; the
+benchmarks where we hit the fast path for literals (in particular
+the two HTML benchmarks and the protobuf benchmark) give very nice
+speedups, and the others are not really affected.
+
+However, benchmarks with Google's GCC branch on other hardware
+is much less clear. It seems that we have a weak loss in some cases
+(and the win for the “typical” win cases are not nearly as clear),
+but that it depends on microarchitecture and plain luck in how we run
+the benchmark. Looking at the generated assembler, it seems that
+the removal of the if causes other large-scale changes in how the
+function is laid out, which makes it likely that this is just bad luck.
+
+Thus, we should keep this change, even though its exact current impact is
+unclear; it's a sensible change per se, and dropping it on the basis of
+microoptimization for a given compiler (or even branch of a compiler)
+would seem like a bad strategy in the long run.
+
+Microbenchmark results (all in 64-bit, opt mode):
+
+ Nehalem, Google GCC:
+
+ Benchmark Base (ns) New (ns) Improvement
+ ------------------------------------------------------------------------------
+ BM_UFlat/0 76747 75591 1.3GB/s html +1.5%
+ BM_UFlat/1 765756 757040 886.3MB/s urls +1.2%
+ BM_UFlat/2 10867 10893 10.9GB/s jpg -0.2%
+ BM_UFlat/3 124 131 1.4GB/s jpg_200 -5.3%
+ BM_UFlat/4 31663 31596 2.8GB/s pdf +0.2%
+ BM_UFlat/5 314162 308176 1.2GB/s html4 +1.9%
+ BM_UFlat/6 29668 29746 790.6MB/s cp -0.3%
+ BM_UFlat/7 12958 13386 796.4MB/s c -3.2%
+ BM_UFlat/8 3596 3682 966.0MB/s lsp -2.3%
+ BM_UFlat/9 1019193 1033493 953.3MB/s xls -1.4%
+ BM_UFlat/10 239 247 775.3MB/s xls_200 -3.2%
+ BM_UFlat/11 236411 240271 606.9MB/s txt1 -1.6%
+ BM_UFlat/12 206639 209768 571.2MB/s txt2 -1.5%
+ BM_UFlat/13 627803 635722 641.4MB/s txt3 -1.2%
+ BM_UFlat/14 845932 857816 538.2MB/s txt4 -1.4%
+ BM_UFlat/15 402107 391670 1.2GB/s bin +2.7%
+ BM_UFlat/16 283 279 683.6MB/s bin_200 +1.4%
+ BM_UFlat/17 46070 46815 781.5MB/s sum -1.6%
+ BM_UFlat/18 5053 5163 782.0MB/s man -2.1%
+ BM_UFlat/19 79721 76581 1.4GB/s pb +4.1%
+ BM_UFlat/20 251158 252330 697.5MB/s gaviota -0.5%
+ Sum of all benchmarks 4966150 4980396 -0.3%
+
+
+ Sandy Bridge, Google GCC:
+
+ Benchmark Base (ns) New (ns) Improvement
+ ------------------------------------------------------------------------------
+ BM_UFlat/0 42850 42182 2.3GB/s html +1.6%
+ BM_UFlat/1 525660 515816 1.3GB/s urls +1.9%
+ BM_UFlat/2 7173 7283 16.3GB/s jpg -1.5%
+ BM_UFlat/3 92 91 2.1GB/s jpg_200 +1.1%
+ BM_UFlat/4 15147 14872 5.9GB/s pdf +1.8%
+ BM_UFlat/5 199936 192116 2.0GB/s html4 +4.1%
+ BM_UFlat/6 12796 12443 1.8GB/s cp +2.8%
+ BM_UFlat/7 6588 6400 1.6GB/s c +2.9%
+ BM_UFlat/8 2010 1951 1.8GB/s lsp +3.0%
+ BM_UFlat/9 761124 763049 1.3GB/s xls -0.3%
+ BM_UFlat/10 186 189 1016.1MB/s xls_200 -1.6%
+ BM_UFlat/11 159354 158460 918.6MB/s txt1 +0.6%
+ BM_UFlat/12 139732 139950 856.1MB/s txt2 -0.2%
+ BM_UFlat/13 429917 425027 961.7MB/s txt3 +1.2%
+ BM_UFlat/14 585255 587324 785.8MB/s txt4 -0.4%
+ BM_UFlat/15 276186 266173 1.8GB/s bin +3.8%
+ BM_UFlat/16 205 207 925.5MB/s bin_200 -1.0%
+ BM_UFlat/17 24925 24935 1.4GB/s sum -0.0%
+ BM_UFlat/18 2632 2576 1.5GB/s man +2.2%
+ BM_UFlat/19 40546 39108 2.8GB/s pb +3.7%
+ BM_UFlat/20 175803 168209 1048.9MB/s gaviota +4.5%
+ Sum of all benchmarks 3408117 3368361 +1.2%
+
+
+ Haswell, upstream GCC 4.8.1:
+
+ Benchmark Base (ns) New (ns) Improvement
+ ------------------------------------------------------------------------------
+ BM_UFlat/0 46308 40641 2.3GB/s html +13.9%
+ BM_UFlat/1 513385 514706 1.3GB/s urls -0.3%
+ BM_UFlat/2 6197 6151 19.2GB/s jpg +0.7%
+ BM_UFlat/3 61 61 3.0GB/s jpg_200 +0.0%
+ BM_UFlat/4 13551 13429 6.5GB/s pdf +0.9%
+ BM_UFlat/5 198317 190243 2.0GB/s html4 +4.2%
+ BM_UFlat/6 14768 12560 1.8GB/s cp +17.6%
+ BM_UFlat/7 6453 6447 1.6GB/s c +0.1%
+ BM_UFlat/8 1991 1980 1.8GB/s lsp +0.6%
+ BM_UFlat/9 766947 770424 1.2GB/s xls -0.5%
+ BM_UFlat/10 170 169 1.1GB/s xls_200 +0.6%
+ BM_UFlat/11 164350 163554 888.7MB/s txt1 +0.5%
+ BM_UFlat/12 145444 143830 832.1MB/s txt2 +1.1%
+ BM_UFlat/13 437849 438413 929.2MB/s txt3 -0.1%
+ BM_UFlat/14 603587 605309 759.8MB/s txt4 -0.3%
+ BM_UFlat/15 249799 248067 1.9GB/s bin +0.7%
+ BM_UFlat/16 191 188 1011.4MB/s bin_200 +1.6%
+ BM_UFlat/17 26064 24778 1.4GB/s sum +5.2%
+ BM_UFlat/18 2620 2601 1.5GB/s man +0.7%
+ BM_UFlat/19 44551 37373 3.0GB/s pb +19.2%
+ BM_UFlat/20 165408 164584 1.0GB/s gaviota +0.5%
+ Sum of all benchmarks 3408011 3385508 +0.7%
+
+------------------------------------------------------------------------
+r77 | snappy.mirrorbot@gmail.com | 2013-06-14 23:42:26 +0200 (Fri, 14 Jun 2013) | 92 lines
+
+Make the two IncrementalCopy* functions take in an ssize_t instead of a len,
+in order to avoid having to do 32-to-64-bit signed conversions on a hot path
+during decompression. (Also fixes some MSVC warnings, mentioned in public
+issue 75, but more of those remain.) They cannot be size_t because we expect
+them to go negative and test for that.
+
+This saves a few movzwl instructions, yielding ~2% speedup in decompression.
+
+
+Sandy Bridge:
+
+Benchmark Base (ns) New (ns) Improvement
+-------------------------------------------------------------------------------------------------
+BM_UFlat/0 48009 41283 2.3GB/s html +16.3%
+BM_UFlat/1 531274 513419 1.3GB/s urls +3.5%
+BM_UFlat/2 7378 7062 16.8GB/s jpg +4.5%
+BM_UFlat/3 92 92 2.0GB/s jpg_200 +0.0%
+BM_UFlat/4 15057 14974 5.9GB/s pdf +0.6%
+BM_UFlat/5 204323 193140 2.0GB/s html4 +5.8%
+BM_UFlat/6 13282 12611 1.8GB/s cp +5.3%
+BM_UFlat/7 6511 6504 1.6GB/s c +0.1%
+BM_UFlat/8 2014 2030 1.7GB/s lsp -0.8%
+BM_UFlat/9 775909 768336 1.3GB/s xls +1.0%
+BM_UFlat/10 182 184 1043.2MB/s xls_200 -1.1%
+BM_UFlat/11 167352 161630 901.2MB/s txt1 +3.5%
+BM_UFlat/12 147393 142246 842.8MB/s txt2 +3.6%
+BM_UFlat/13 449960 432853 944.4MB/s txt3 +4.0%
+BM_UFlat/14 620497 594845 775.9MB/s txt4 +4.3%
+BM_UFlat/15 265610 267356 1.8GB/s bin -0.7%
+BM_UFlat/16 206 205 932.7MB/s bin_200 +0.5%
+BM_UFlat/17 25561 24730 1.4GB/s sum +3.4%
+BM_UFlat/18 2620 2644 1.5GB/s man -0.9%
+BM_UFlat/19 45766 38589 2.9GB/s pb +18.6%
+BM_UFlat/20 171107 169832 1039.5MB/s gaviota +0.8%
+Sum of all benchmarks 3500103 3394565 +3.1%
+
+
+Westmere:
+
+Benchmark Base (ns) New (ns) Improvement
+-------------------------------------------------------------------------------------------------
+BM_UFlat/0 72624 71526 1.3GB/s html +1.5%
+BM_UFlat/1 735821 722917 930.8MB/s urls +1.8%
+BM_UFlat/2 10450 10172 11.7GB/s jpg +2.7%
+BM_UFlat/3 117 117 1.6GB/s jpg_200 +0.0%
+BM_UFlat/4 29817 29648 3.0GB/s pdf +0.6%
+BM_UFlat/5 297126 293073 1.3GB/s html4 +1.4%
+BM_UFlat/6 28252 27994 842.0MB/s cp +0.9%
+BM_UFlat/7 12672 12391 862.1MB/s c +2.3%
+BM_UFlat/8 3507 3425 1040.9MB/s lsp +2.4%
+BM_UFlat/9 1004268 969395 1018.0MB/s xls +3.6%
+BM_UFlat/10 233 227 844.8MB/s xls_200 +2.6%
+BM_UFlat/11 230054 224981 647.8MB/s txt1 +2.3%
+BM_UFlat/12 201229 196447 610.5MB/s txt2 +2.4%
+BM_UFlat/13 609547 596761 685.3MB/s txt3 +2.1%
+BM_UFlat/14 824362 804821 573.8MB/s txt4 +2.4%
+BM_UFlat/15 371095 374899 1.3GB/s bin -1.0%
+BM_UFlat/16 267 267 717.8MB/s bin_200 +0.0%
+BM_UFlat/17 44623 43828 835.9MB/s sum +1.8%
+BM_UFlat/18 5077 4815 841.0MB/s man +5.4%
+BM_UFlat/19 74964 73210 1.5GB/s pb +2.4%
+BM_UFlat/20 237987 236745 746.0MB/s gaviota +0.5%
+Sum of all benchmarks 4794092 4697659 +2.1%
+
+
+Istanbul:
+
+Benchmark Base (ns) New (ns) Improvement
+-------------------------------------------------------------------------------------------------
+BM_UFlat/0 98614 96376 1020.4MB/s html +2.3%
+BM_UFlat/1 963740 953241 707.2MB/s urls +1.1%
+BM_UFlat/2 25042 24769 4.8GB/s jpg +1.1%
+BM_UFlat/3 180 180 1065.6MB/s jpg_200 +0.0%
+BM_UFlat/4 45942 45403 1.9GB/s pdf +1.2%
+BM_UFlat/5 400135 390226 1008.2MB/s html4 +2.5%
+BM_UFlat/6 37768 37392 631.9MB/s cp +1.0%
+BM_UFlat/7 18585 18200 588.2MB/s c +2.1%
+BM_UFlat/8 5751 5690 627.7MB/s lsp +1.1%
+BM_UFlat/9 1543154 1542209 641.4MB/s xls +0.1%
+BM_UFlat/10 381 388 494.6MB/s xls_200 -1.8%
+BM_UFlat/11 339715 331973 440.1MB/s txt1 +2.3%
+BM_UFlat/12 294807 289418 415.4MB/s txt2 +1.9%
+BM_UFlat/13 906160 884094 463.3MB/s txt3 +2.5%
+BM_UFlat/14 1224221 1198435 386.1MB/s txt4 +2.2%
+BM_UFlat/15 516277 502923 979.5MB/s bin +2.7%
+BM_UFlat/16 405 402 477.2MB/s bin_200 +0.7%
+BM_UFlat/17 61640 60621 605.6MB/s sum +1.7%
+BM_UFlat/18 7326 7383 549.5MB/s man -0.8%
+BM_UFlat/19 94720 92653 1.2GB/s pb +2.2%
+BM_UFlat/20 360435 346687 510.6MB/s gaviota +4.0%
+Sum of all benchmarks 6944998 6828663 +1.7%
+
+------------------------------------------------------------------------
+r76 | snappy.mirrorbot@gmail.com | 2013-06-13 18:19:52 +0200 (Thu, 13 Jun 2013) | 9 lines
+
+Add support for uncompressing to iovecs (scatter I/O).
+Windows does not have struct iovec defined anywhere,
+so we define our own version that's equal to what UNIX
+typically has.
+
+The bulk of this patch was contributed by Mohit Aron.
+
+R=jeff
+
+------------------------------------------------------------------------
+r75 | snappy.mirrorbot@gmail.com | 2013-06-12 21:51:15 +0200 (Wed, 12 Jun 2013) | 4 lines
+
+Some code reorganization needed for an internal change.
+
+R=fikes
+
+------------------------------------------------------------------------
+r74 | snappy.mirrorbot@gmail.com | 2013-04-09 17:33:30 +0200 (Tue, 09 Apr 2013) | 4 lines
+
+Supports truncated test data in zippy benchmark.
+
+R=sesse
+
+------------------------------------------------------------------------
+r73 | snappy.mirrorbot@gmail.com | 2013-02-05 15:36:15 +0100 (Tue, 05 Feb 2013) | 4 lines
+
+Release Snappy 1.1.0.
+
+R=sanjay
+
+------------------------------------------------------------------------
r72 | snappy.mirrorbot@gmail.com | 2013-02-05 15:30:05 +0100 (Tue, 05 Feb 2013) | 9 lines
Make ./snappy_unittest pass without "srcdir" being defined.
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/NEWS new/snappy-1.1.1/NEWS
--- old/snappy-1.1.0/NEWS 2013-02-05 15:36:32.000000000 +0100
+++ new/snappy-1.1.1/NEWS 2013-10-15 12:41:47.000000000 +0200
@@ -1,3 +1,16 @@
+Snappy v1.1.1, October 15th 2013:
+
+ * Add support for uncompressing to iovecs (scatter I/O).
+ The bulk of this patch was contributed by Mohit Aron.
+
+ * Speed up decompression by ~2%; much more so (~13-20%) on
+ a few benchmarks on given compilers and CPUs.
+
+ * Fix a few issues with MSVC compilation.
+
+ * Support truncated test data in the benchmark.
+
+
Snappy v1.1.0, January 18th 2013:
* Snappy now uses 64 kB block size instead of 32 kB. On average,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/config.h.in new/snappy-1.1.1/config.h.in
--- old/snappy-1.1.0/config.h.in 2013-02-05 15:37:42.000000000 +0100
+++ new/snappy-1.1.1/config.h.in 2013-10-15 12:43:26.000000000 +0200
@@ -1,8 +1,5 @@
/* config.h.in. Generated from configure.ac by autoheader. */
-/* Define if building universal (internal helper macro) */
-#undef AC_APPLE_UNIVERSAL_BUILD
-
/* Define to 1 if the compiler supports __builtin_ctz and friends. */
#undef HAVE_BUILTIN_CTZ
@@ -103,9 +100,6 @@
/* Define to the one symbol short name of this package. */
#undef PACKAGE_TARNAME
-/* Define to the home page for this package. */
-#undef PACKAGE_URL
-
/* Define to the version of this package. */
#undef PACKAGE_VERSION
@@ -116,13 +110,15 @@
#undef VERSION
/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
- significant byte first (like Motorola and SPARC, unlike Intel). */
-#if defined AC_APPLE_UNIVERSAL_BUILD
-# if defined __BIG_ENDIAN__
-# define WORDS_BIGENDIAN 1
-# endif
-#else
-# ifndef WORDS_BIGENDIAN
-# undef WORDS_BIGENDIAN
-# endif
+ significant byte first (like Motorola and SPARC, unlike Intel and VAX). */
+#if defined __BIG_ENDIAN__
+# define WORDS_BIGENDIAN 1
+#elif ! defined __LITTLE_ENDIAN__
+# undef WORDS_BIGENDIAN
#endif
+
+/* Define to `unsigned int' if does not define. */
+#undef size_t
+
+/* Define to `int' if does not define. */
+#undef ssize_t
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/configure.ac new/snappy-1.1.1/configure.ac
--- old/snappy-1.1.0/configure.ac 2013-02-05 15:36:32.000000000 +0100
+++ new/snappy-1.1.1/configure.ac 2013-10-15 12:41:48.000000000 +0200
@@ -1,11 +1,11 @@
m4_define([snappy_major], [1])
m4_define([snappy_minor], [1])
-m4_define([snappy_patchlevel], [0])
+m4_define([snappy_patchlevel], [1])
# Libtool shared library interface versions (current:revision:age)
# Update this value for every release! (A:B:C will map to foo.so.(A-C).C.B)
# http://www.gnu.org/software/libtool/manual/html_node/Updating-version-info.h...
-m4_define([snappy_ltversion], [2:4:1])
+m4_define([snappy_ltversion], [3:0:2])
AC_INIT([snappy], [snappy_major.snappy_minor.snappy_patchlevel])
AC_CONFIG_MACRO_DIR([m4])
@@ -18,6 +18,8 @@
AC_PROG_CXX
AC_LANG([C++])
AC_C_BIGENDIAN
+AC_TYPE_SIZE_T
+AC_TYPE_SSIZE_T
AC_CHECK_HEADERS([stdint.h stddef.h sys/mman.h sys/resource.h windows.h byteswap.h sys/byteswap.h sys/endian.h sys/time.h])
# Don't use AC_FUNC_MMAP, as it checks for mappings of already-mapped memory,
@@ -110,6 +112,11 @@
else
AC_SUBST([ac_cv_have_stddef_h], [0])
fi
+if test "$ac_cv_header_sys_uio_h" = "yes"; then
+ AC_SUBST([ac_cv_have_sys_uio_h], [1])
+else
+ AC_SUBST([ac_cv_have_sys_uio_h], [0])
+fi
# Export the version to snappy-stubs-public.h.
SNAPPY_MAJOR="snappy_major"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/snappy-stubs-public.h new/snappy-1.1.1/snappy-stubs-public.h
--- old/snappy-1.1.0/snappy-stubs-public.h 2013-02-05 15:37:42.000000000 +0100
+++ new/snappy-1.1.1/snappy-stubs-public.h 2013-10-15 12:43:25.000000000 +0200
@@ -44,9 +44,13 @@
#include
#endif
+#if 0
+#include
+#endif
+
#define SNAPPY_MAJOR 1
#define SNAPPY_MINOR 1
-#define SNAPPY_PATCHLEVEL 0
+#define SNAPPY_PATCHLEVEL 1
#define SNAPPY_VERSION \
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
@@ -80,6 +84,15 @@
TypeName(const TypeName&); \
void operator=(const TypeName&)
+#if !0
+// Windows does not have an iovec type, yet the concept is universally useful.
+// It is simple to define it ourselves, so we put it inside our own namespace.
+struct iovec {
+ void* iov_base;
+ size_t iov_len;
+};
+#endif
+
} // namespace snappy
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/snappy-stubs-public.h.in new/snappy-1.1.1/snappy-stubs-public.h.in
--- old/snappy-1.1.0/snappy-stubs-public.h.in 2013-01-04 12:53:57.000000000 +0100
+++ new/snappy-1.1.1/snappy-stubs-public.h.in 2013-10-15 12:41:48.000000000 +0200
@@ -44,6 +44,10 @@
#include
#endif
+#if @ac_cv_have_sys_uio_h@
+#include
+#endif
+
#define SNAPPY_MAJOR @SNAPPY_MAJOR@
#define SNAPPY_MINOR @SNAPPY_MINOR@
#define SNAPPY_PATCHLEVEL @SNAPPY_PATCHLEVEL@
@@ -80,6 +84,15 @@
TypeName(const TypeName&); \
void operator=(const TypeName&)
+#if !@ac_cv_have_sys_uio_h@
+// Windows does not have an iovec type, yet the concept is universally useful.
+// It is simple to define it ourselves, so we put it inside our own namespace.
+struct iovec {
+ void* iov_base;
+ size_t iov_len;
+};
+#endif
+
} // namespace snappy
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/snappy-test.cc new/snappy-1.1.1/snappy-test.cc
--- old/snappy-1.1.0/snappy-test.cc 2013-02-05 15:36:32.000000000 +0100
+++ new/snappy-1.1.1/snappy-test.cc 2013-10-15 12:41:48.000000000 +0200
@@ -42,21 +42,25 @@
namespace snappy {
-string ReadTestDataFile(const string& base) {
+string ReadTestDataFile(const string& base, size_t size_limit) {
string contents;
const char* srcdir = getenv("srcdir"); // This is set by Automake.
+ string prefix;
if (srcdir) {
- file::ReadFileToString(string(srcdir) + "/testdata/" + base,
- &contents,
- file::Defaults()).CheckSuccess();
- } else {
- file::ReadFileToString("testdata/" + base,
- &contents,
- file::Defaults()).CheckSuccess();
+ prefix = string(srcdir) + "/";
+ }
+ file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults()
+ ).CheckSuccess();
+ if (size_limit > 0) {
+ contents = contents.substr(0, size_limit);
}
return contents;
}
+string ReadTestDataFile(const string& base) {
+ return ReadTestDataFile(base, 0);
+}
+
string StringPrintf(const char* format, ...) {
char buf[4096];
va_list ap;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/snappy-test.h new/snappy-1.1.1/snappy-test.h
--- old/snappy-1.1.0/snappy-test.h 2013-02-05 15:36:32.000000000 +0100
+++ new/snappy-1.1.1/snappy-test.h 2013-10-15 12:41:48.000000000 +0200
@@ -139,10 +139,10 @@
void CheckSuccess() { }
};
- DummyStatus ReadFileToString(const char* filename, string* data, int unused) {
- FILE* fp = fopen(filename, "rb");
+ DummyStatus GetContents(const string& filename, string* data, int unused) {
+ FILE* fp = fopen(filename.c_str(), "rb");
if (fp == NULL) {
- perror(filename);
+ perror(filename.c_str());
exit(1);
}
@@ -160,15 +160,9 @@
fclose(fp);
}
- DummyStatus ReadFileToString(const string& filename,
- string* data,
- int unused) {
- ReadFileToString(filename.c_str(), data, unused);
- }
-
- DummyStatus WriteStringToFile(const string& str,
- const string& filename,
- int unused) {
+ DummyStatus SetContents(const string& filename,
+ const string& str,
+ int unused) {
FILE* fp = fopen(filename.c_str(), "wb");
if (fp == NULL) {
perror(filename.c_str());
@@ -203,6 +197,8 @@
void Test_Snappy_FindMatchLength();
void Test_Snappy_FindMatchLengthRandom();
+string ReadTestDataFile(const string& base, size_t size_limit);
+
string ReadTestDataFile(const string& base);
// A sprintf() variant that returns a std::string.
@@ -332,6 +328,7 @@
(new Benchmark(#benchmark_name, benchmark_name))
extern Benchmark* Benchmark_BM_UFlat;
+extern Benchmark* Benchmark_BM_UIOVec;
extern Benchmark* Benchmark_BM_UValidate;
extern Benchmark* Benchmark_BM_ZFlat;
@@ -482,6 +479,7 @@
fprintf(stderr, "---------------------------------------------------\n");
snappy::Benchmark_BM_UFlat->Run();
+ snappy::Benchmark_BM_UIOVec->Run();
snappy::Benchmark_BM_UValidate->Run();
snappy::Benchmark_BM_ZFlat->Run();
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/snappy.cc new/snappy-1.1.1/snappy.cc
--- old/snappy-1.1.0/snappy.cc 2013-01-04 12:53:32.000000000 +0100
+++ new/snappy-1.1.1/snappy.cc 2013-10-15 12:41:48.000000000 +0200
@@ -82,6 +82,7 @@
COPY_2_BYTE_OFFSET = 2,
COPY_4_BYTE_OFFSET = 3
};
+static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset.
// Copy "len" bytes from "src" to "op", one byte at a time. Used for
// handling COPY operations where the input and output regions may
@@ -94,7 +95,7 @@
// ababababababababababab
// Note that this does not match the semantics of either memcpy()
// or memmove().
-static inline void IncrementalCopy(const char* src, char* op, int len) {
+static inline void IncrementalCopy(const char* src, char* op, ssize_t len) {
assert(len > 0);
do {
*op++ = *src++;
@@ -136,9 +137,7 @@
const int kMaxIncrementCopyOverflow = 10;
-} // namespace
-
-static inline void IncrementalCopyFastPath(const char* src, char* op, int len) {
+inline void IncrementalCopyFastPath(const char* src, char* op, ssize_t len) {
while (op - src < 8) {
UnalignedCopy64(src, op);
len -= op - src;
@@ -152,6 +151,8 @@
}
}
+} // namespace
+
static inline char* EmitLiteral(char* op,
const char* literal,
int len,
@@ -469,21 +470,26 @@
// bool Append(const char* ip, size_t length);
// bool AppendFromSelf(uint32 offset, size_t length);
//
-// // The difference between TryFastAppend and Append is that TryFastAppend
-// // is allowed to read up to <available> bytes from the input buffer,
-// // whereas Append is allowed to read <length>.
+// // The rules for how TryFastAppend differs from Append are somewhat
+// // convoluted:
// //
-// // Also, TryFastAppend is allowed to return false, declining the append,
-// // without it being a fatal error -- just "return false" would be
-// // a perfectly legal implementation of TryFastAppend. The intention
-// // is for TryFastAppend to allow a fast path in the common case of
-// // a small append.
+// // - TryFastAppend is allowed to decline (return false) at any
+// // time, for any reason -- just "return false" would be
+// // a perfectly legal implementation of TryFastAppend.
+// // The intention is for TryFastAppend to allow a fast path
+// // in the common case of a small append.
+// // - TryFastAppend is allowed to read up to <available> bytes
+// // from the input buffer, whereas Append is allowed to read
+// // <length>. However, if it returns true, it must leave
+// // at least five (kMaximumTagLength) bytes in the input buffer
+// // afterwards, so that there is always enough space to read the
+// // next tag without checking for a refill.
+// // - TryFastAppend must always return decline (return false)
+// // if <length> is 61 or more, as in this case the literal length is not
+// // decoded fully. In practice, this should not be a big problem,
+// // as it is unlikely that one would implement a fast path accepting
+// // this much data.
// //
-// // NOTE(user): TryFastAppend must always return decline (return false)
-// // if <length> is 61 or more, as in this case the literal length is not
-// // decoded fully. In practice, this should not be a big problem,
-// // as it is unlikely that one would implement a fast path accepting
-// // this much data.
// bool TryFastAppend(const char* ip, size_t available, size_t length);
// };
@@ -652,7 +658,7 @@
const char* ip_limit_; // Points just past buffered bytes
uint32 peeked_; // Bytes peeked from reader (need to skip)
bool eof_; // Hit end of input without an error?
- char scratch_[5]; // Temporary buffer for PeekFast() boundaries
+ char scratch_[kMaximumTagLength]; // See RefillTag().
// Ensure that all of the tag metadata for the next tag is available
// in [ip_..ip_limit_-1]. Also ensures that [ip,ip+4] is readable even
@@ -715,7 +721,7 @@
// scope to optimize the expression based on the local
// context, which overall increases speed.
#define MAYBE_REFILL() \
- if (ip_limit_ - ip < 5) { \
+ if (ip_limit_ - ip < kMaximumTagLength) { \
ip_ = ip; \
if (!RefillTag()) return; \
ip = ip_; \
@@ -730,7 +736,9 @@
if (writer->TryFastAppend(ip, ip_limit_ - ip, literal_length)) {
assert(literal_length < 61);
ip += literal_length;
- MAYBE_REFILL();
+ // NOTE(user): There is no MAYBE_REFILL() here, as TryFastAppend()
+ // will not return true unless there's already at least five spare
+ // bytes in addition to the literal.
continue;
}
if (PREDICT_FALSE(literal_length >= 61)) {
@@ -823,7 +831,7 @@
assert(nbuf == needed);
ip_ = scratch_;
ip_limit_ = scratch_ + needed;
- } else if (nbuf < 5) {
+ } else if (nbuf < kMaximumTagLength) {
// Have enough bytes, but move into scratch_ so that we do not
// read past end of input
memmove(scratch_, ip, nbuf);
@@ -839,27 +847,18 @@
}
template <typename Writer>
-static bool InternalUncompress(Source* r,
- Writer* writer,
- uint32 max_len) {
+static bool InternalUncompress(Source* r, Writer* writer) {
// Read the uncompressed length from the front of the compressed input
SnappyDecompressor decompressor(r);
uint32 uncompressed_len = 0;
if (!decompressor.ReadUncompressedLength(&uncompressed_len)) return false;
- return InternalUncompressAllTags(
- &decompressor, writer, uncompressed_len, max_len);
+ return InternalUncompressAllTags(&decompressor, writer, uncompressed_len);
}
template <typename Writer>
static bool InternalUncompressAllTags(SnappyDecompressor* decompressor,
Writer* writer,
- uint32 uncompressed_len,
- uint32 max_len) {
- // Protect against possible DoS attack
- if (static_cast<uint64>(uncompressed_len) > max_len) {
- return false;
- }
-
+ uint32 uncompressed_len) {
writer->SetExpectedLength(uncompressed_len);
// Process the entire input
@@ -954,6 +953,183 @@
}
// -----------------------------------------------------------------------
+// IOVec interfaces
+// -----------------------------------------------------------------------
+
+// A type that writes to an iovec.
+// Note that this is not a "ByteSink", but a type that matches the
+// Writer template argument to SnappyDecompressor::DecompressAllTags().
+class SnappyIOVecWriter {
+ private:
+ const struct iovec* output_iov_;
+ const size_t output_iov_count_;
+
+ // We are currently writing into output_iov_[curr_iov_index_].
+ int curr_iov_index_;
+
+ // Bytes written to output_iov_[curr_iov_index_] so far.
+ size_t curr_iov_written_;
+
+ // Total bytes decompressed into output_iov_ so far.
+ size_t total_written_;
+
+ // Maximum number of bytes that will be decompressed into output_iov_.
+ size_t output_limit_;
+
+ inline char* GetIOVecPointer(int index, size_t offset) {
+ return reinterpret_cast(output_iov_[index].iov_base) +
+ offset;
+ }
+
+ public:
+ // Does not take ownership of iov. iov must be valid during the
+ // entire lifetime of the SnappyIOVecWriter.
+ inline SnappyIOVecWriter(const struct iovec* iov, size_t iov_count)
+ : output_iov_(iov),
+ output_iov_count_(iov_count),
+ curr_iov_index_(0),
+ curr_iov_written_(0),
+ total_written_(0),
+ output_limit_(-1) {
+ }
+
+ inline void SetExpectedLength(size_t len) {
+ output_limit_ = len;
+ }
+
+ inline bool CheckLength() const {
+ return total_written_ == output_limit_;
+ }
+
+ inline bool Append(const char* ip, size_t len) {
+ if (total_written_ + len > output_limit_) {
+ return false;
+ }
+
+ while (len > 0) {
+ assert(curr_iov_written_ <= output_iov_[curr_iov_index_].iov_len);
+ if (curr_iov_written_ >= output_iov_[curr_iov_index_].iov_len) {
+ // This iovec is full. Go to the next one.
+ if (curr_iov_index_ + 1 >= output_iov_count_) {
+ return false;
+ }
+ curr_iov_written_ = 0;
+ ++curr_iov_index_;
+ }
+
+ const size_t to_write = std::min(
+ len, output_iov_[curr_iov_index_].iov_len - curr_iov_written_);
+ memcpy(GetIOVecPointer(curr_iov_index_, curr_iov_written_),
+ ip,
+ to_write);
+ curr_iov_written_ += to_write;
+ total_written_ += to_write;
+ ip += to_write;
+ len -= to_write;
+ }
+
+ return true;
+ }
+
+ inline bool TryFastAppend(const char* ip, size_t available, size_t len) {
+ const size_t space_left = output_limit_ - total_written_;
+ if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16 &&
+ output_iov_[curr_iov_index_].iov_len - curr_iov_written_ >= 16) {
+ // Fast path, used for the majority (about 95%) of invocations.
+ char* ptr = GetIOVecPointer(curr_iov_index_, curr_iov_written_);
+ UnalignedCopy64(ip, ptr);
+ UnalignedCopy64(ip + 8, ptr + 8);
+ curr_iov_written_ += len;
+ total_written_ += len;
+ return true;
+ }
+
+ return false;
+ }
+
+ inline bool AppendFromSelf(size_t offset, size_t len) {
+ if (offset > total_written_ || offset == 0) {
+ return false;
+ }
+ const size_t space_left = output_limit_ - total_written_;
+ if (len > space_left) {
+ return false;
+ }
+
+ // Locate the iovec from which we need to start the copy.
+ int from_iov_index = curr_iov_index_;
+ size_t from_iov_offset = curr_iov_written_;
+ while (offset > 0) {
+ if (from_iov_offset >= offset) {
+ from_iov_offset -= offset;
+ break;
+ }
+
+ offset -= from_iov_offset;
+ --from_iov_index;
+ assert(from_iov_index >= 0);
+ from_iov_offset = output_iov_[from_iov_index].iov_len;
+ }
+
+ // Copy <len> bytes starting from the iovec pointed to by from_iov_index to
+ // the current iovec.
+ while (len > 0) {
+ assert(from_iov_index <= curr_iov_index_);
+ if (from_iov_index != curr_iov_index_) {
+ const size_t to_copy = std::min(
+ output_iov_[from_iov_index].iov_len - from_iov_offset,
+ len);
+ Append(GetIOVecPointer(from_iov_index, from_iov_offset), to_copy);
+ len -= to_copy;
+ if (len > 0) {
+ ++from_iov_index;
+ from_iov_offset = 0;
+ }
+ } else {
+ assert(curr_iov_written_ <= output_iov_[curr_iov_index_].iov_len);
+ size_t to_copy = std::min(output_iov_[curr_iov_index_].iov_len -
+ curr_iov_written_,
+ len);
+ if (to_copy == 0) {
+ // This iovec is full. Go to the next one.
+ if (curr_iov_index_ + 1 >= output_iov_count_) {
+ return false;
+ }
+ ++curr_iov_index_;
+ curr_iov_written_ = 0;
+ continue;
+ }
+ if (to_copy > len) {
+ to_copy = len;
+ }
+ IncrementalCopy(GetIOVecPointer(from_iov_index, from_iov_offset),
+ GetIOVecPointer(curr_iov_index_, curr_iov_written_),
+ to_copy);
+ curr_iov_written_ += to_copy;
+ from_iov_offset += to_copy;
+ total_written_ += to_copy;
+ len -= to_copy;
+ }
+ }
+
+ return true;
+ }
+
+};
+
+bool RawUncompressToIOVec(const char* compressed, size_t compressed_length,
+ const struct iovec* iov, size_t iov_cnt) {
+ ByteArraySource reader(compressed, compressed_length);
+ return RawUncompressToIOVec(&reader, iov, iov_cnt);
+}
+
+bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov,
+ size_t iov_cnt) {
+ SnappyIOVecWriter output(iov, iov_cnt);
+ return InternalUncompress(compressed, &output);
+}
+
+// -----------------------------------------------------------------------
// Flat array interfaces
// -----------------------------------------------------------------------
@@ -994,7 +1170,7 @@
inline bool TryFastAppend(const char* ip, size_t available, size_t len) {
char* op = op_;
const size_t space_left = op_limit_ - op;
- if (len <= 16 && available >= 16 && space_left >= 16) {
+ if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16) {
// Fast path, used for the majority (about 95%) of invocations.
UnalignedCopy64(ip, op);
UnalignedCopy64(ip + 8, op + 8);
@@ -1009,7 +1185,16 @@
char* op = op_;
const size_t space_left = op_limit_ - op;
- if (op - base_ <= offset - 1u) { // -1u catches offset==0
+ // Check if we try to append from before the start of the buffer.
+ // Normally this would just be a check for "produced < offset",
+ // but "produced <= offset - 1u" is equivalent for every case
+ // except the one where offset==0, where the right side will wrap around
+ // to a very big number. This is convenient, as offset==0 is another
+ // invalid case that we also want to catch, so that we do not go
+ // into an infinite loop.
+ assert(op >= base_);
+ size_t produced = op - base_;
+ if (produced <= offset - 1u) {
return false;
}
if (len <= 16 && offset >= 8 && space_left >= 16) {
@@ -1039,7 +1224,7 @@
bool RawUncompress(Source* compressed, char* uncompressed) {
SnappyArrayWriter output(uncompressed);
- return InternalUncompress(compressed, &output, kuint32max);
+ return InternalUncompress(compressed, &output);
}
bool Uncompress(const char* compressed, size_t n, string* uncompressed) {
@@ -1047,9 +1232,9 @@
if (!GetUncompressedLength(compressed, n, &ulength)) {
return false;
}
- // Protect against possible DoS attack
- if ((static_cast<uint64>(ulength) + uncompressed->size()) >
- uncompressed->max_size()) {
+ // On 32-bit builds: max_size() < kuint32max. Check for that instead
+ // of crashing (e.g., consider externally specified compressed data).
+ if (ulength > uncompressed->max_size()) {
return false;
}
STLStringResizeUninitialized(uncompressed, ulength);
@@ -1079,7 +1264,9 @@
return false;
}
inline bool AppendFromSelf(size_t offset, size_t len) {
- if (produced_ <= offset - 1u) return false; // -1u catches offset==0
+ // See SnappyArrayWriter::AppendFromSelf for an explanation of
+ // the "offset - 1u" trick.
+ if (produced_ <= offset - 1u) return false;
produced_ += len;
return produced_ <= expected_;
}
@@ -1088,7 +1275,7 @@
bool IsValidCompressedBuffer(const char* compressed, size_t n) {
ByteArraySource reader(compressed, n);
SnappyDecompressionValidator writer;
- return InternalUncompress(&reader, &writer, kuint32max);
+ return InternalUncompress(&reader, &writer);
}
void RawCompress(const char* input,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/snappy.h new/snappy-1.1.1/snappy.h
--- old/snappy-1.1.0/snappy.h 2013-02-05 15:36:32.000000000 +0100
+++ new/snappy-1.1.1/snappy.h 2013-10-15 12:41:48.000000000 +0200
@@ -124,6 +124,28 @@
// returns false if the message is corrupted and could not be decrypted
bool RawUncompress(Source* compressed, char* uncompressed);
+ // Given data in "compressed[0..compressed_length-1]" generated by
+ // calling the Snappy::Compress routine, this routine
+ // stores the uncompressed data to the iovec "iov". The number of physical
+ // buffers in "iov" is given by iov_cnt and their cumulative size
+ // must be at least GetUncompressedLength(compressed). The individual buffers
+ // in "iov" must not overlap with each other.
+ //
+ // returns false if the message is corrupted and could not be decrypted
+ bool RawUncompressToIOVec(const char* compressed, size_t compressed_length,
+ const struct iovec* iov, size_t iov_cnt);
+
+ // Given data from the byte source 'compressed' generated by calling
+ // the Snappy::Compress routine, this routine stores the uncompressed
+ // data to the iovec "iov". The number of physical
+ // buffers in "iov" is given by iov_cnt and their cumulative size
+ // must be at least GetUncompressedLength(compressed). The individual buffers
+ // in "iov" must not overlap with each other.
+ //
+ // returns false if the message is corrupted and could not be decrypted
+ bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov,
+ size_t iov_cnt);
+
// Returns the maximal size of the compressed representation of
// input data that is "source_bytes" bytes in length;
size_t MaxCompressedLength(size_t source_bytes);
@@ -156,7 +178,6 @@
static const int kMaxHashTableBits = 14;
static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
-
} // end namespace snappy
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' --exclude Makefile.in --exclude configure --exclude config.guess --exclude '*.pot' --exclude mkinstalldirs --exclude aclocal.m4 --exclude config.sub --exclude depcomp --exclude install-sh --exclude ltmain.sh old/snappy-1.1.0/snappy_unittest.cc new/snappy-1.1.1/snappy_unittest.cc
--- old/snappy-1.1.0/snappy_unittest.cc 2013-02-05 15:36:32.000000000 +0100
+++ new/snappy-1.1.1/snappy_unittest.cc 2013-10-15 12:41:48.000000000 +0200
@@ -492,6 +492,46 @@
}
+static void VerifyIOVec(const string& input) {
+ string compressed;
+ DataEndingAtUnreadablePage i(input);
+ const size_t written = snappy::Compress(i.data(), i.size(), &compressed);
+ CHECK_EQ(written, compressed.size());
+ CHECK_LE(compressed.size(),
+ snappy::MaxCompressedLength(input.size()));
+ CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
+
+ // Try uncompressing into an iovec containing a random number of entries
+ // ranging from 1 to 10.
+ char* buf = new char[input.size()];
+ ACMRandom rnd(input.size());
+ int num = rnd.Next() % 10 + 1;
+ if (input.size() < num) {
+ num = input.size();
+ }
+ struct iovec* iov = new iovec[num];
+ int used_so_far = 0;
+ for (int i = 0; i < num; ++i) {
+ iov[i].iov_base = buf + used_so_far;
+ if (i == num - 1) {
+ iov[i].iov_len = input.size() - used_so_far;
+ } else {
+ // Randomly choose to insert a 0 byte entry.
+ if (rnd.OneIn(5)) {
+ iov[i].iov_len = 0;
+ } else {
+ iov[i].iov_len = rnd.Uniform(input.size());
+ }
+ }
+ used_so_far += iov[i].iov_len;
+ }
+ CHECK(snappy::RawUncompressToIOVec(
+ compressed.data(), compressed.size(), iov, num));
+ CHECK(!memcmp(buf, input.data(), input.size()));
+ delete[] iov;
+ delete[] buf;
+}
+
// Test that data compressed by a compressor that does not
// obey block sizes is uncompressed properly.
static void VerifyNonBlockedCompression(const string& input) {
@@ -542,8 +582,11 @@
VerifyNonBlockedCompression(input);
+ VerifyIOVec(input);
if (!input.empty()) {
- VerifyNonBlockedCompression(Expand(input));
+ const string expanded = Expand(input);
+ VerifyNonBlockedCompression(expanded);
+ VerifyIOVec(input);
}
@@ -611,7 +654,8 @@
// try reading stuff in from a bad file.
for (int i = 1; i <= 3; ++i) {
- string data = ReadTestDataFile(StringPrintf("baddata%d.snappy", i).c_str());
+ string data = ReadTestDataFile(StringPrintf("baddata%d.snappy", i).c_str(),
+ 0);
string uncmp;
// check that we don't return a crazy length
size_t ulen;
@@ -663,7 +707,7 @@
}
length -= to_copy;
- if ((to_copy < 12) && (offset < 2048)) {
+ if ((to_copy >= 4) && (to_copy < 12) && (offset < 2048)) {
assert(to_copy-4 < 8); // Must fit in 3 bits
dst->push_back(1 | ((to_copy-4) << 2) | ((offset >> 8) << 5));
dst->push_back(offset & 0xff);
@@ -768,10 +812,123 @@
string uncompressed;
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
- CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncompressed));
+ CHECK(snappy::Uncompress(compressed.data(), compressed.size(),
+ &uncompressed));
CHECK_EQ(uncompressed, src);
}
+TEST(Snappy, IOVecEdgeCases) {
+ // Test some tricky edge cases in the iovec output that are not necessarily
+ // exercised by random tests.
+
+ // Our output blocks look like this initially (the last iovec is bigger
+ // than depicted):
+ // [ ] [ ] [ ] [ ] [ ]
+ static const int kLengths[] = { 2, 1, 4, 8, 128 };
+
+ struct iovec iov[ARRAYSIZE(kLengths)];
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ iov[i].iov_base = new char[kLengths[i]];
+ iov[i].iov_len = kLengths[i];
+ }
+
+ string compressed;
+ Varint::Append32(&compressed, 22);
+
+ // A literal whose output crosses three blocks.
+ // [ab] [c] [123 ] [ ] [ ]
+ AppendLiteral(&compressed, "abc123");
+
+ // A copy whose output crosses two blocks (source and destination
+ // segments marked).
+ // [ab] [c] [1231] [23 ] [ ]
+ // ^--^ --
+ AppendCopy(&compressed, 3, 3);
+
+ // A copy where the input is, at first, in the block before the output:
+ //
+ // [ab] [c] [1231] [231231 ] [ ]
+ // ^--- ^---
+ // Then during the copy, the pointers move such that the input and
+ // output pointers are in the same block:
+ //
+ // [ab] [c] [1231] [23123123] [ ]
+ // ^- ^-
+ // And then they move again, so that the output pointer is no longer
+ // in the same block as the input pointer:
+ // [ab] [c] [1231] [23123123] [123 ]
+ // ^-- ^--
+ AppendCopy(&compressed, 6, 9);
+
+ // Finally, a copy where the input is from several blocks back,
+ // and it also crosses three blocks:
+ //
+ // [ab] [c] [1231] [23123123] [123b ]
+ // ^ ^
+ // [ab] [c] [1231] [23123123] [123bc ]
+ // ^ ^
+ // [ab] [c] [1231] [23123123] [123bc12 ]
+ // ^- ^-
+ AppendCopy(&compressed, 17, 4);
+
+ CHECK(snappy::RawUncompressToIOVec(
+ compressed.data(), compressed.size(), iov, ARRAYSIZE(iov)));
+ CHECK_EQ(0, memcmp(iov[0].iov_base, "ab", 2));
+ CHECK_EQ(0, memcmp(iov[1].iov_base, "c", 1));
+ CHECK_EQ(0, memcmp(iov[2].iov_base, "1231", 4));
+ CHECK_EQ(0, memcmp(iov[3].iov_base, "23123123", 8));
+ CHECK_EQ(0, memcmp(iov[4].iov_base, "123bc12", 7));
+
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ delete[] reinterpret_cast(iov[i].iov_base);
+ }
+}
+
+TEST(Snappy, IOVecLiteralOverflow) {
+ static const int kLengths[] = { 3, 4 };
+
+ struct iovec iov[ARRAYSIZE(kLengths)];
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ iov[i].iov_base = new char[kLengths[i]];
+ iov[i].iov_len = kLengths[i];
+ }
+
+ string compressed;
+ Varint::Append32(&compressed, 8);
+
+ AppendLiteral(&compressed, "12345678");
+
+ CHECK(!snappy::RawUncompressToIOVec(
+ compressed.data(), compressed.size(), iov, ARRAYSIZE(iov)));
+
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ delete[] reinterpret_cast(iov[i].iov_base);
+ }
+}
+
+TEST(Snappy, IOVecCopyOverflow) {
+ static const int kLengths[] = { 3, 4 };
+
+ struct iovec iov[ARRAYSIZE(kLengths)];
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ iov[i].iov_base = new char[kLengths[i]];
+ iov[i].iov_len = kLengths[i];
+ }
+
+ string compressed;
+ Varint::Append32(&compressed, 8);
+
+ AppendLiteral(&compressed, "123");
+ AppendCopy(&compressed, 3, 5);
+
+ CHECK(!snappy::RawUncompressToIOVec(
+ compressed.data(), compressed.size(), iov, ARRAYSIZE(iov)));
+
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ delete[] reinterpret_cast(iov[i].iov_base);
+ }
+}
+
static bool CheckUncompressedLength(const string& compressed,
size_t* ulength) {
@@ -971,19 +1128,18 @@
static void CompressFile(const char* fname) {
string fullinput;
- file::ReadFileToString(fname, &fullinput, file::Defaults()).CheckSuccess();
+ file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
string compressed;
Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false);
- file::WriteStringToFile(
- string(fname).append(".comp").c_str(), compressed,
- file::Defaults()).CheckSuccess();
+ file::SetContents(string(fname).append(".comp"), compressed, file::Defaults())
+ .CheckSuccess();
}
static void UncompressFile(const char* fname) {
string fullinput;
- file::ReadFileToString(fname, &fullinput, file::Defaults()).CheckSuccess();
+ file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
size_t uncompLength;
CHECK(CheckUncompressedLength(fullinput, &uncompLength));
@@ -992,14 +1148,13 @@
uncompressed.resize(uncompLength);
CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed));
- file::WriteStringToFile(
- string(fname).append(".uncomp").c_str(), uncompressed,
- file::Defaults()).CheckSuccess();
+ file::SetContents(string(fname).append(".uncomp"), uncompressed,
+ file::Defaults()).CheckSuccess();
}
static void MeasureFile(const char* fname) {
string fullinput;
- file::ReadFileToString(fname, &fullinput, file::Defaults()).CheckSuccess();
+ file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
printf("%-40s :\n", fname);
int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len;
@@ -1032,25 +1187,29 @@
static struct {
const char* label;
const char* filename;
+ size_t size_limit;
} files[] = {
- { "html", "html" },
- { "urls", "urls.10K" },
- { "jpg", "house.jpg" },
- { "pdf", "mapreduce-osdi-1.pdf" },
- { "html4", "html_x_4" },
- { "cp", "cp.html" },
- { "c", "fields.c" },
- { "lsp", "grammar.lsp" },
- { "xls", "kennedy.xls" },
- { "txt1", "alice29.txt" },
- { "txt2", "asyoulik.txt" },
- { "txt3", "lcet10.txt" },
- { "txt4", "plrabn12.txt" },
- { "bin", "ptt5" },
- { "sum", "sum" },
- { "man", "xargs.1" },
- { "pb", "geo.protodata" },
- { "gaviota", "kppkn.gtb" },
+ { "html", "html", 0 },
+ { "urls", "urls.10K", 0 },
+ { "jpg", "house.jpg", 0 },
+ { "jpg_200", "house.jpg", 200 },
+ { "pdf", "mapreduce-osdi-1.pdf", 0 },
+ { "html4", "html_x_4", 0 },
+ { "cp", "cp.html", 0 },
+ { "c", "fields.c", 0 },
+ { "lsp", "grammar.lsp", 0 },
+ { "xls", "kennedy.xls", 0 },
+ { "xls_200", "kennedy.xls", 200 },
+ { "txt1", "alice29.txt", 0 },
+ { "txt2", "asyoulik.txt", 0 },
+ { "txt3", "lcet10.txt", 0 },
+ { "txt4", "plrabn12.txt", 0 },
+ { "bin", "ptt5", 0 },
+ { "bin_200", "ptt5", 200 },
+ { "sum", "sum", 0 },
+ { "man", "xargs.1", 0 },
+ { "pb", "geo.protodata", 0 },
+ { "gaviota", "kppkn.gtb", 0 },
};
static void BM_UFlat(int iters, int arg) {
@@ -1059,7 +1218,8 @@
// Pick file to process based on "arg"
CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files));
- string contents = ReadTestDataFile(files[arg].filename);
+ string contents = ReadTestDataFile(files[arg].filename,
+ files[arg].size_limit);
string zcontents;
snappy::Compress(contents.data(), contents.size(), &zcontents);
@@ -1076,7 +1236,7 @@
delete[] dst;
}
-BENCHMARK(BM_UFlat)->DenseRange(0, 17);
+BENCHMARK(BM_UFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
static void BM_UValidate(int iters, int arg) {
StopBenchmarkTiming();
@@ -1084,7 +1244,8 @@
// Pick file to process based on "arg"
CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files));
- string contents = ReadTestDataFile(files[arg].filename);
+ string contents = ReadTestDataFile(files[arg].filename,
+ files[arg].size_limit);
string zcontents;
snappy::Compress(contents.data(), contents.size(), &zcontents);
@@ -1100,6 +1261,52 @@
}
BENCHMARK(BM_UValidate)->DenseRange(0, 4);
+static void BM_UIOVec(int iters, int arg) {
+ StopBenchmarkTiming();
+
+ // Pick file to process based on "arg"
+ CHECK_GE(arg, 0);
+ CHECK_LT(arg, ARRAYSIZE(files));
+ string contents = ReadTestDataFile(files[arg].filename,
+ files[arg].size_limit);
+
+ string zcontents;
+ snappy::Compress(contents.data(), contents.size(), &zcontents);
+
+ // Uncompress into an iovec containing ten entries.
+ const int kNumEntries = 10;
+ struct iovec iov[kNumEntries];
+ char *dst = new char[contents.size()];
+ int used_so_far = 0;
+ for (int i = 0; i < kNumEntries; ++i) {
+ iov[i].iov_base = dst + used_so_far;
+ if (used_so_far == contents.size()) {
+ iov[i].iov_len = 0;
+ continue;
+ }
+
+ if (i == kNumEntries - 1) {
+ iov[i].iov_len = contents.size() - used_so_far;
+ } else {
+ iov[i].iov_len = contents.size() / kNumEntries;
+ }
+ used_so_far += iov[i].iov_len;
+ }
+
+ SetBenchmarkBytesProcessed(static_cast<int64>(iters) *
+ static_cast<int64>(contents.size()));
+ SetBenchmarkLabel(files[arg].label);
+ StartBenchmarkTiming();
+ while (iters-- > 0) {
+ CHECK(snappy::RawUncompressToIOVec(zcontents.data(), zcontents.size(), iov,
+ kNumEntries));
+ }
+ StopBenchmarkTiming();
+
+ delete[] dst;
+}
+BENCHMARK(BM_UIOVec)->DenseRange(0, 4);
+
static void BM_ZFlat(int iters, int arg) {
StopBenchmarkTiming();
@@ -1107,7 +1314,8 @@
// Pick file to process based on "arg"
CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files));
- string contents = ReadTestDataFile(files[arg].filename);
+ string contents = ReadTestDataFile(files[arg].filename,
+ files[arg].size_limit);
char* dst = new char[snappy::MaxCompressedLength(contents.size())];
@@ -1128,7 +1336,7 @@
files[arg].label, contents.size(), zsize);
delete[] dst;
}
-BENCHMARK(BM_ZFlat)->DenseRange(0, 17);
+BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
} // namespace snappy
--
To unsubscribe, e-mail: opensuse-commit+unsubscribe@opensuse.org
For additional commands, e-mail: opensuse-commit+help@opensuse.org