[yast-commit] r65471 - in /trunk/packager: ./ package/ src/modules/ testsuite/tests/

Author: lslezak Date: Wed Aug 31 14:55:07 2011 New Revision: 65471 URL: http://svn.opensuse.org/viewcvs/yast?rev=65471&view=rev Log: - SpaceCalculation - improved target installation size estimation, consider also filesystem properties (journal size, reserved space) add some (static) non-package files to disk usage, increased some spare size constants (bnc#263275) - 2.21.12 Added: trunk/packager/testsuite/tests/SpaceCalculation.err trunk/packager/testsuite/tests/SpaceCalculation.out trunk/packager/testsuite/tests/SpaceCalculation.ycp Modified: trunk/packager/package/yast2-packager.changes trunk/packager/src/modules/SpaceCalculation.ycp trunk/packager/yast2-packager.spec.in Modified: trunk/packager/package/yast2-packager.changes URL: http://svn.opensuse.org/viewcvs/yast/trunk/packager/package/yast2-packager.changes?rev=65471&r1=65470&r2=65471&view=diff ============================================================================== --- trunk/packager/package/yast2-packager.changes (original) +++ trunk/packager/package/yast2-packager.changes Wed Aug 31 14:55:07 2011 @@ -1,9 +1,17 @@ ------------------------------------------------------------------- +Wed Aug 31 12:21:00 UTC 2011 - lslezak@suse.cz + +- SpaceCalculation - improved target installation size estimation, + consider also filesystem properties (journal size, reserved space) + add some (static) non-package files to disk usage, increased + some spare size constants (bnc#263275) +- 2.21.12 + +------------------------------------------------------------------- Wed Aug 31 14:05:48 CEST 2011 - locilka@suse.cz - Cleaner solution for adjusting repository priority and fixing that now it also works in installation (bnc#714027) -- 2.21.12 ------------------------------------------------------------------- Tue Aug 30 14:03:01 UTC 2011 - lslezak@suse.cz Modified: trunk/packager/src/modules/SpaceCalculation.ycp URL: http://svn.opensuse.org/viewcvs/yast/trunk/packager/src/modules/SpaceCalculation.ycp?rev=65471&r1=65470&r2=65471&view=diff ============================================================================== --- trunk/packager/src/modules/SpaceCalculation.ycp (original) +++ trunk/packager/src/modules/SpaceCalculation.ycp Wed Aug 31 14:55:07 2011 @@ -93,7 +93,7 @@ list > partition = []; // the sizes are in kB - integer min_spare = 1 * 1024; // 1 MB + integer min_spare = 10 * 1024; // 10 MB integer max_spare = 1024 * 1024; // 1 GB string target = Installation::destdir; @@ -221,6 +221,295 @@ return part_input; }; + // return default ext3/4 journal size (in B) for target partition size + integer DefaultExtJournalSize(map part) + { + if (part["used_fs"]:`unknown == `ext2) + { + y2milestone("No journal on ext2"); + return 0; + } + + integer ret = 0; + + integer part_size = 1024 * part["size_k"]:0; + // default block size is 4k + integer bs = tointeger(part["fs_options","opt_blocksize", "option_value"]:"4096"); + integer blocks = part_size / bs; + + y2milestone("Partition %1: %2 blocks (block size: %3)", part["name"]:"", blocks, bs); + + // values extracted from ext2fs_default_journal_size() function in e2fsprogs sources + if (blocks < 2048) + { + ret = 0; + } + else if (blocks < 32768) + { + ret = 1024; + } + else if (blocks < 256*1024) + { + ret = 4096; + } + else if (blocks < 512*1024) + { + ret = 8192; + } + else if (blocks < 1024*1024) + { + ret = 16384; + } + else + { + // maximum journal size + ret = 32768; + } + + // converts blocks to bytes + ret = ret * bs; + + y2milestone("Default journal size: %1kB", ret / 1024); + + + return ret; + } + + integer ExtJournalSize(map part) + { + if (part["used_fs"]:`unknown == `ext2) + { + y2milestone("No journal on ext2"); + return 0; + } + + integer ret = 0; + // no journal + if (haskey(part["fs_options"]:$[], "no_journal")) + { + y2milestone("Partition %1 has disabled journal", part["name"]:""); + } + // default journal size for ext3/4 + else + { + y2milestone("Using default journal size for %1", part["name"]:""); + ret = DefaultExtJournalSize(part); + } + // Note: custom journal size cannot be entered in the partitioner + + y2milestone("Journal size for %1: %2kB", part["name"]:"", ret / 1024); + + return ret; + } + + integer XfsJournalSize(map part) + { + integer part_size = 1024 * part["size_k"]:0; + integer mb = 1 << 20; + integer gb = 1 << 30; + + // the default log size to fs size ratio is 1:2048 + // (the value is then adjusted according to many other fs parameters, + // we take just the simple approach here, it should be sufficient) + integer ret = part_size / 2048; + + // check min and max limits + integer min_log_size = 10 * mb; + integer max_log_size = 2 * gb; + + if (ret < min_log_size) + { + ret = min_log_size; + } + else if (ret > max_log_size) + { + ret = max_log_size; + } + + y2milestone("Estimated journal size for XFS partition %1kB: %2kB", part_size / 1024, ret / 1024); + + return ret; + } + + integer ReiserJournalSize(map part) + { + // the default is 8193 of 4k blocks (max = 32749, min = 513 blocks) + integer ret = 8193 * 4096; + + y2milestone("Default Reiser journal size: %1kB", ret / 1024); + + return ret; + } + + integer DefaultJfsJournalSize(integer part_size) + { + // the default is 0.4% rounded to megabytes, 128MB max. + integer ret = part_size >> 8; // 0.4% ~= 1/256 + integer max = 128 * (1 << 20); // 128 MB + + ret = ((ret + (1 << 20) - 1) >> 20) << 20; + + if (ret > max) + { + ret = max; + } + + y2milestone("Default JFS journal size: %1MB", ret >> 20); + + return ret; + } + + integer JfsJournalSize(map part) + { + // log size (in MB) + integer log_size = tointeger(part["fs_options","opt_log_size", "option_value"]:"0"); + + if (log_size > 0) + { + // convert to bytes + log_size = log_size * (1 << 20); + } + else + { + log_size = DefaultJfsJournalSize(1024 * part["size_k"]:0); + } + + y2milestone("Jfs journal size: %1MB", log_size >> 20); + + return log_size; + } + + list<map> EstimateTargetUsage(list<map> parts) + { + y2milestone("EstimateTargetUsage(%1)", parts); + integer mb = 1 << 10; // sizes are in kB, 1MB is 1024 kB + + // invalid or empty input + if (parts == nil || size(parts) == 0) + { + y2error("Invalid input: %1", parts); + return []; + } + + // the numbers are from openSUSE-11.4 default KDE installation + map used_mapping = $[ + "/var/lib/rpm" : 42 * mb, // RPM database + "/var/log" : 14 * mb, // system logs (YaST logs have ~12MB) + "/var/adm/backup" : 10 * mb, // backups + "/var/cache/zypp" : 38 * mb, // zypp metadata cache after refresh (with OSS + update repos) + "/etc" : 2 * mb, // various /etc config files not belonging to any package + "/usr/share" : 1 * mb, // some files created by postinstall scripts + "/boot/initrd" : 11 * mb // depends on HW but better than nothing + ]; + + y2milestone("Adding target size mapping: %1", used_mapping); + + list<string> mount_points = []; + + // convert list to map indexed by mount point + map mounts = listmap(map part, parts, { + mount_points = add(mount_points, part["name"]:""); + return $[ part["name"]:"" : part ]; + } + ); + + + foreach(string dir, integer used, used_mapping, { + string mounted = String::FindMountPoint(dir, mount_points); + y2milestone("Dir %1 is mounted on %2", dir, mounted); + + map part = mounts[mounted]:$[]; + + if (part != $[]) + { + integer curr_used = part["used"]:0; + y2milestone("Adding %1kB to %2kB currently used", used, curr_used); + curr_used = curr_used + used; + + part["used"] = curr_used; + part["free"] = part["free"]:0 - used; + + mounts[mounted] = part; + } + else + { + y2warning("Cannot find partition for mount point %1, ignoring it", mounted); + } + } + ); + + // convert back to list + list<map> ret = maplist(string dir, map part, mounts, {return part;}); + + y2milestone("EstimateTargetUsage() result: %1", ret); + + return ret; + } + + // is the filesystem one of Ext2/3/4? + boolean ExtFs(symbol fs) + { + return fs == `ext2 || fs == `ext3 || fs == `ext4; + } + + // return estimated fs overhead + // (the difference between partition size and reported fs blocks) + integer EstimateFsOverhead(map part) + { + integer fs_size = 1024 * part["size_k"]:0; + symbol fs = part["used_fs"]:`unknown; + + integer ret = 0; + + if (ExtFs(fs)) + { + // ext2/3/4 overhead is about 1.6% according to my test (8GB partition) + ret = fs_size * 16 / 1000; + y2milestone("Estimated Ext2/3/4 overhead: %1kB", ret); + } + else if (fs == `xfs) + { + // xfs overhead is about 0.1% + ret = fs_size / 1000; + y2milestone("Estimated XFS overhead: %1kB", ret); + } + else if (fs == `jfs) + { + // jfs overhead is about 0.3% + ret = fs_size * 3 / 1000; + y2milestone("Estimated JFS overhead: %1kB", ret); + } + // reiser and btrfs have negligible overhead, just ignore it + + return ret; + } + + // return reserved space for root user (in bytes) + integer ReservedSpace(map part) + { + // read the percentage + string option = part["fs_options", "opt_reserved_blocks", "option_value"]:""; + integer ret = 0; + + if (option != nil && option != "") + { + float percent = tofloat(option); + + if (percent > 0.0) + { + // convert to absolute value + integer fs_size = part["size_k"]:0; + ret = tointeger(fs_size / 100 * percent); + } + } + + if (ret > 0) + { + y2milestone("Partition %1: reserved space: %2%% (%3kB)", part["name"]:"", option, ret); + } + + return ret * 1024; + } /* * Define a macro that transforms information about all partitions ( from @@ -286,7 +575,7 @@ } list<map> target_partitions = []; - integer min_spare = 5 * 1024 * 1024; // minimum free space ( 5 MB ) + integer min_spare = 20 * 1024 * 1024; // minimum free space ( 20 MB ) foreach( string disk, map diskinfo, targets, ``{ @@ -376,6 +665,56 @@ continue; } } + else + // for formatted partitions estimate free system size + { + // compute fs overhead + used = EstimateFsOverhead(part); + + if (used > 0) + { + y2milestone("Partition %1: assuming fs overhead: %2kB", part["device"]:"", used / 1024); + } + + // journal size + integer js = 0; + + if (ExtFs(used_fs)) + { + js = ExtJournalSize(part); + integer reserved = ReservedSpace(part); + + if (reserved > 0) + { + used = used + reserved; + } + } + else if (used_fs == `xfs) + { + js = XfsJournalSize(part); + } + else if (used_fs == `reiser) + { + js = ReiserJournalSize(part); + } + else if (used_fs == `jfs) + { + js = JfsJournalSize(part); + } + else + { + y2warning("Unknown journal size for filesystem: %1", used_fs); + } + + if (js > 0) + { + y2milestone("Partition %1: assuming journal size: %2kB", part["device"]:"", js / 1024); + used = used + js; + } + + // decrease free size + free_size = free_size - used; + } // convert into kB for TargetInitDU free_size = free_size / 1024; @@ -409,6 +748,9 @@ } ); // foreach (`part) } ); // foreach (`disk) + // add estimated size occupied by non-package files + target_partitions = EstimateTargetUsage(target_partitions); + y2milestone( "get_partition_info: part %1", target_partitions ); Pkg::TargetInitDU (target_partitions); @@ -440,7 +782,7 @@ } else if ( Mode::update () ) { - partition = EvaluateFreeSpace ( 5 ); // 5% free spare for update/upgrade + partition = EvaluateFreeSpace ( 15 ); // 15% free spare for update/upgrade } else if ( Mode::normal () ) { Added: trunk/packager/testsuite/tests/SpaceCalculation.err URL: http://svn.opensuse.org/viewcvs/yast/trunk/packager/testsuite/tests/SpaceCalculation.err?rev=65471&view=auto ============================================================================== (empty) Added: trunk/packager/testsuite/tests/SpaceCalculation.out URL: http://svn.opensuse.org/viewcvs/yast/trunk/packager/testsuite/tests/SpaceCalculation.out?rev=65471&view=auto ============================================================================== --- trunk/packager/testsuite/tests/SpaceCalculation.out (added) +++ trunk/packager/testsuite/tests/SpaceCalculation.out Wed Aug 31 14:55:07 2011 @@ -0,0 +1,42 @@ +Read .target.tmpdir "/tmp" +Dump ----- Journal size tests ----- +Dump Ext2/3/4 journal size tests +Return 0 +Return 134217728 +Return 134217728 +Return 0 +Return 67108864 +Return 33554432 +Return 0 +Dump ReiserFS journal size tests +Return 33558528 +Dump XFS journal size tests +Return 10485760 +Return 26214400 +Return 2147483648 +Dump JFS journal size tests +Return 20971520 +Return 22020096 +Return 134217728 +Return 10485760 +Dump ----- Extfs reserved space tests ----- +Return 0 +Return 0 +Return 2684354560 +Return 6710886400 +Dump ----- Fs overhead tests ----- +Return 85899345 +Return 85899345 +Return 85899345 +Return 5368709 +Return 16106127 +Return 0 +Return 0 +Dump ----- Target usage tests ----- +Log Invalid input: nil +Return [] +Log Invalid input: [] +Return [] +Return [$["free":9879168, "name":"/", "used":120832]] +Return [$["free":9879168, "name":"/", "used":120832], $["free":1000000, "name":"/home", "used":0]] +Return [$["free":9891456, "name":"/", "used":108544], $["free":988736, "name":"/boot", "used":11264], $["free":998976, "name":"/usr", "used":1024]] Added: trunk/packager/testsuite/tests/SpaceCalculation.ycp URL: http://svn.opensuse.org/viewcvs/yast/trunk/packager/testsuite/tests/SpaceCalculation.ycp?rev=65471&view=auto ============================================================================== --- trunk/packager/testsuite/tests/SpaceCalculation.ycp (added) +++ trunk/packager/testsuite/tests/SpaceCalculation.ycp Wed Aug 31 14:55:07 2011 @@ -0,0 +1,178 @@ +/** + * Testsuite for SpaceCalculation.ycp module + * + */ + +{ + +include "testsuite.ycp"; +map READ = $[ + "target" : $[ + "tmpdir" : "/tmp" + ] +]; +TESTSUITE_INIT ([READ], nil); + +import "SpaceCalculation"; + +// size units - multiplies of kB blocks +integer mb = 1 << 10; +integer gb = 1 << 20; +integer tb = 1 << 30; + +map part = $[ + // 5GB + "size_k" : 5 * gb, + "used_fs" : `ext2, + "name" : "sda1" +]; + +DUMP(" ----- Journal size tests ----- "); +DUMP("Ext2/3/4 journal size tests"); + +// ext2 => no journal => 0 +TEST(``(SpaceCalculation::ExtJournalSize(part)), [], nil); + +// 128MB +part["used_fs"] = `ext3; +TEST(``(SpaceCalculation::ExtJournalSize(part)), [], nil); + +// 128MB +part["used_fs"] = `ext4; +TEST(``(SpaceCalculation::ExtJournalSize(part)), [], nil); + +// 2 MB is too small => 0 +part["size_k"] = 2 * mb; +TEST(``(SpaceCalculation::ExtJournalSize(part)), [], nil); + +// 2GB => 64MB +part["size_k"] = 2 * gb; +TEST(``(SpaceCalculation::ExtJournalSize(part)), [], nil); + +// 2GB but 1k blocks => 32MB +part["fs_options"] = $["opt_blocksize" : $[ "option_value" : "1024" ]]; +TEST(``(SpaceCalculation::ExtJournalSize(part)), [], nil); + +// no journal option => 0 +part["fs_options"] = $["no_journal" : $[ "option_value" : true ]]; +TEST(``(SpaceCalculation::ExtJournalSize(part)), [], nil); + + +DUMP("ReiserFS journal size tests"); + +part["fs_options"] = $[]; +part["used_fs"] = `reiser; + +// the default is 32MB + 4kB regardeless fs size +TEST(``(SpaceCalculation::ReiserJournalSize(part)), [], nil); + + +DUMP("XFS journal size tests"); + +part["used_fs"] = `xfs; + +// too small => 10 MB min size +TEST(``(SpaceCalculation::XfsJournalSize(part)), [], nil); + +// medium size => 26MB +part["size_k"] = 50 * gb; +TEST(``(SpaceCalculation::XfsJournalSize(part)), [], nil); + +// too large => 2GB max size +part["size_k"] = 5 * tb; +TEST(``(SpaceCalculation::XfsJournalSize(part)), [], nil); + + + +DUMP("JFS journal size tests"); + +part["used_fs"] = `jfs; + +// medium size +part["size_k"] = 5 * gb; +TEST(``(SpaceCalculation::JfsJournalSize(part)), [], nil); + +// medium size, add few kB more so it's rounded one MB up +part["size_k"] = (5 * gb) + 5; +TEST(``(SpaceCalculation::JfsJournalSize(part)), [], nil); + +// too large => 128MB max size +part["size_k"] = 50 * gb; +TEST(``(SpaceCalculation::JfsJournalSize(part)), [], nil); + +// user defined size (in MB) +part["fs_options"] = $["opt_log_size" : $[ "option_value" : "10" ]]; +TEST(``(SpaceCalculation::JfsJournalSize(part)), [], nil); +part["fs_options"] = $[]; + +DUMP(" ----- Extfs reserved space tests ----- "); + +// no reserved space +TEST(``(SpaceCalculation::ReservedSpace(part)), [], nil); + +// 0% +part["fs_options"] = $["opt_reserved_blocks" : $[ "option_value" : "0.0" ]]; +TEST(``(SpaceCalculation::ReservedSpace(part)), [], nil); + +// 5% of 50GB => 2.5GB +part["fs_options"] = $["opt_reserved_blocks" : $[ "option_value" : "5.0" ]]; +TEST(``(SpaceCalculation::ReservedSpace(part)), [], nil); + +// 12.50% of 50GB => 6.25GB +part["fs_options"] = $["opt_reserved_blocks" : $[ "option_value" : "12.50" ]]; +TEST(``(SpaceCalculation::ReservedSpace(part)), [], nil); + +DUMP(" ----- Fs overhead tests ----- "); + +// 5GB partition +part["size_k"] = 5 * gb; + +// ext2 +part["used_fs"] = `ext2; +TEST(``(SpaceCalculation::EstimateFsOverhead(part)), [], nil); + +// ext3 +part["used_fs"] = `ext3; +TEST(``(SpaceCalculation::EstimateFsOverhead(part)), [], nil); + +// ext4 +part["used_fs"] = `ext4; +TEST(``(SpaceCalculation::EstimateFsOverhead(part)), [], nil); + +// xfs +part["used_fs"] = `xfs; +TEST(``(SpaceCalculation::EstimateFsOverhead(part)), [], nil); + +// jfs +part["used_fs"] = `jfs; +TEST(``(SpaceCalculation::EstimateFsOverhead(part)), [], nil); + +// reiser +part["used_fs"] = `reiser; +TEST(``(SpaceCalculation::EstimateFsOverhead(part)), [], nil); + +// btrfs +part["used_fs"] = `btrfs; +TEST(``(SpaceCalculation::EstimateFsOverhead(part)), [], nil); + + + +DUMP(" ----- Target usage tests ----- "); + +// test invalid input +TEST(``(SpaceCalculation::EstimateTargetUsage(nil)), [], nil); +TEST(``(SpaceCalculation::EstimateTargetUsage([])), [], nil); + +// single partition +TEST(``(SpaceCalculation::EstimateTargetUsage([$["name":"/", "used":0, "free":10000000]])), [], nil); + +// multiple partitions, separate /home (nothing to install) +TEST(``(SpaceCalculation::EstimateTargetUsage([$["name":"/", "used":0, "free":10000000], + $["name":"/home", "used":0, "free":1000000]])), [], nil); + +// multiple partitions +TEST(``(SpaceCalculation::EstimateTargetUsage([$["name":"/", "used":0, "free":10000000], + $["name":"/boot", "used":0, "free":1000000], + $["name":"/usr", "used":0, "free":1000000]])), [], nil); + +} Modified: trunk/packager/yast2-packager.spec.in URL: http://svn.opensuse.org/viewcvs/yast/trunk/packager/yast2-packager.spec.in?rev=65471&r1=65470&r2=65471&view=diff ============================================================================== --- trunk/packager/yast2-packager.spec.in (original) +++ trunk/packager/yast2-packager.spec.in Wed Aug 31 14:55:07 2011 @@ -9,8 +9,8 @@ # HTTP.ycp BuildRequires: yast2-transfer -# Installation summary in PackagesUI:: -BuildRequires: yast2 >= 2.18.1 +# String::FindMountPoint() +BuildRequires: yast2 >= 2.21.13 # Pkg::SetZConfig() BuildRequires: yast2-pkg-bindings >= 2.21.8 @@ -21,8 +21,8 @@ # Pkg::SetZConfig() Requires: yast2-pkg-bindings >= 2.21.8 -# Installation summary in PackagesUI:: -Requires: yast2 >= 2.18.1 +# String::FindMountPoint() +Requires: yast2 >= 2.21.13 # unzipping license file Requires: unzip -- To unsubscribe, e-mail: yast-commit+unsubscribe@opensuse.org For additional commands, e-mail: yast-commit+help@opensuse.org