Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package MirrorCache for openSUSE:Factory checked in at 2024-08-01 22:05:36 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old) and /work/SRC/openSUSE:Factory/.MirrorCache.new.7232 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "MirrorCache" Thu Aug 1 22:05:36 2024 rev:43 rq:1190907 version:1.079 Changes: -------- --- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2024-06-27 16:05:08.234656119 +0200 +++ /work/SRC/openSUSE:Factory/.MirrorCache.new.7232/MirrorCache.changes 2024-08-01 22:06:25.245394943 +0200 @@ -1,0 +2,15 @@ +Thu Jul 25 07:40:38 UTC 2024 - Andrii Nikitin <andrii.nikitin@suse.com> + +- Update to version 1.079: + * Show size of projects (#502) + +------------------------------------------------------------------- +Thu Jul 18 08:23:40 UTC 2024 - Andrii Nikitin <andrii.nikitin@suse.com> + +- Update to version 1.078: + * Respect If-Modified-Since header (#501) + * Add content-disposition header in torrent responses (#497) + * Reuse file extention instead of url parameter in redirects (#497) + * Allow caches to serve files while re-checking (#492) + +------------------------------------------------------------------- Old: ---- MirrorCache-1.077.obscpio New: ---- MirrorCache-1.079.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ MirrorCache.spec ++++++ --- /var/tmp/diff_new_pack.Zlvxh3/_old 2024-08-01 22:06:25.873420858 +0200 +++ /var/tmp/diff_new_pack.Zlvxh3/_new 2024-08-01 22:06:25.873420858 +0200 @@ -22,7 +22,7 @@ %define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4 perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801 perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece) perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg) perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base) perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg) perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands) perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX) perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base) perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA) perl(Config::IniFiles) %define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow sysuser-tools Name: MirrorCache -Version: 1.077 +Version: 1.079 Release: 0 Summary: WebApp to redirect and manage mirrors License: GPL-2.0-or-later ++++++ MirrorCache-1.077.obscpio -> MirrorCache-1.079.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/assets/javascripts/admintable.js new/MirrorCache-1.079/assets/javascripts/admintable.js --- old/MirrorCache-1.077/assets/javascripts/admintable.js 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/assets/javascripts/admintable.js 2024-07-24 16:14:18.000000000 +0200 @@ -257,6 +257,14 @@ return htmlEscape(data); } +function renderAdminUnixtime(data, type, row, meta) { + return data ? new Date(data*1000).toISOString() : ''; +} + +function renderAdminReadonly(data, type, row, meta) { + return data ? data : ''; +} + function renderAdminTableSettingsList(data, type, row, meta) { var plainText = type !== 'display'; var edit = isEditingAdminTableRow(meta); @@ -409,6 +417,11 @@ } else { // columnName = th.text().trim().toLowerCase().replace(/ /g, '_'); columnName = th.text().trim().toLowerCase(); + if (columnName == 'file count') { + columnName = 'file cnt'; + } else if (columnName == 'last modified') { + columnName = 'lm'; + } } columns.push({ data: columnName }); @@ -428,6 +441,12 @@ columnDef.render = renderAdminTableValue; } emptyRow[columnName] = ""; + } else if (th.hasClass('col_ro')) { + columnDef.render = renderAdminReadonly; + emptyRow.settings = {}; + } else if (th.hasClass('col_unixtime')) { + columnDef.render = renderAdminUnixtime; + emptyRow.settings = {}; } else if (th.hasClass('col_settings')) { columnDef.render = renderAdminTableSettings; emptyRow.settings = {}; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/Datamodule.pm new/MirrorCache-1.079/lib/MirrorCache/Datamodule.pm --- old/MirrorCache-1.077/lib/MirrorCache/Datamodule.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/Datamodule.pm 2024-07-24 16:14:18.000000000 +0200 @@ -356,8 +356,7 @@ my $param = $c->req->params; if (!$skip_xtra && $self->_original_path =~ m/(\.metalink|\.meta4|\.zsync|\.mirrorlist|\.torrent|\.magnet|\.btih)$/) { $xtra = $1; - $xtra = substr($xtra, 1); - $param->append($xtra => 1); + $url = $url . $xtra; } if (my $version = Directory::Scanner::OBSMediaVersion::parse_version($url)) { $c->res->headers->add('X-MEDIA-VERSION' => $version); @@ -616,8 +615,9 @@ ) { $self->must_render_from_root(1); my $time = ~time() & 0xff; + my $time2 = 60*60 + $time; # allow caches to serve content for 1h while they re-check - $self->c->res->headers->cache_control("public, max-age=$time"); + $self->c->res->headers->cache_control("public, max-age=$time stale-while-revalidate=$time2 stale-if-error=86400"); } my ($ext) = $path =~ /([^.]+)$/; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/Schema/Result/Project.pm new/MirrorCache-1.079/lib/MirrorCache/Schema/Result/Project.pm --- old/MirrorCache-1.077/lib/MirrorCache/Schema/Result/Project.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/Schema/Result/Project.pm 2024-07-24 16:14:18.000000000 +0200 @@ -35,6 +35,9 @@ is_nullable => 1 }, db_sync_full_every => { data_type => "integer", is_nullable => 1 }, + size => { data_type => "bigint", is_nullable => 1 }, + file_cnt => { data_type => "bigint", is_nullable => 1 }, + lm => { data_type => "bigint", is_nullable => 1 }, ); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/Schema/ResultSet/Folder.pm new/MirrorCache-1.079/lib/MirrorCache/Schema/ResultSet/Folder.pm --- old/MirrorCache-1.077/lib/MirrorCache/Schema/ResultSet/Folder.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/Schema/ResultSet/Folder.pm 2024-07-24 16:14:18.000000000 +0200 @@ -404,4 +404,21 @@ $prep->execute($pathfrom, $pathto, $pathto); } +sub calculate_disk_usage { + my ($self, $path) = @_; + + my $rsource = $self->result_source; + my $schema = $rsource->schema; + my $dbh = $schema->storage->dbh; + + $path = $path . "%"; + + my $sql = "select sum(file.size) size, count(*) file_cnt, max(mtime) lm from file join folder on folder_id = folder.id where path like ?"; + my $prep = $dbh->prepare($sql); + $prep->execute($path); + my $res = $dbh->selectrow_hashref($prep); + return ($res->{size}, $res->{file_cnt}, $res->{lm}); +} + + 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/Schema/ResultSet/Project.pm new/MirrorCache-1.079/lib/MirrorCache/Schema/ResultSet/Project.pm --- old/MirrorCache-1.077/lib/MirrorCache/Schema/ResultSet/Project.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/Schema/ResultSet/Project.pm 2024-07-24 16:14:18.000000000 +0200 @@ -22,5 +22,22 @@ $prep->execute($project_id); } +sub update_disk_usage { + my ($self, $path, $size, $file_cnt, $lm) = @_; + + my $rsource = $self->result_source; + my $schema = $rsource->schema; + my $dbh = $schema->storage->dbh; + + my $sql = << "END_SQL"; +update project set +size = ?, +file_cnt = ?, +lm = (case when lm > ? then lm else ? end) +where path = ? +END_SQL + my $prep = $dbh->prepare($sql); + $prep->execute($size, $file_cnt, $lm, $lm, $path); +} 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/Task/ReportProjectSize.pm new/MirrorCache-1.079/lib/MirrorCache/Task/ReportProjectSize.pm --- old/MirrorCache-1.077/lib/MirrorCache/Task/ReportProjectSize.pm 1970-01-01 01:00:00.000000000 +0100 +++ new/MirrorCache-1.079/lib/MirrorCache/Task/ReportProjectSize.pm 2024-07-24 16:14:18.000000000 +0200 @@ -0,0 +1,45 @@ +# Copyright (C) 2024 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +package MirrorCache::Task::ReportProjectSize; +use Mojo::Base 'Mojolicious::Plugin'; + + +sub register { + my ($self, $app) = @_; + $app->minion->add_task(report_project_size => sub { _run($app, @_) }); +} + +sub _run { + my ($app, $job, $path) = @_; + return $job->fail('Empty path is not allowed') unless $path; + + my $minion = $app->minion; + return $job->finish('Previous report job is still active for ' . $path) + unless my $guard = $minion->guard('report_project_size_' . $path, 30*60); + + my $realpath = $app->mc->root->rootpath($path); + return $job->fail('Path not found: ' . $path) unless $realpath; + + my ($size, $file_cnt, $lm) = $app->schema->resultset('Folder')->calculate_disk_usage($path); + + $job->note("total size" => $size, "file count" => $file_cnt, "last modified" => $lm); + + $app->schema->resultset('Project')->update_disk_usage($path, $size, $file_cnt, $lm); + + return $job->finish; +} + +1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/Task/ReportProjectSizeSchedule.pm new/MirrorCache-1.079/lib/MirrorCache/Task/ReportProjectSizeSchedule.pm --- old/MirrorCache-1.077/lib/MirrorCache/Task/ReportProjectSizeSchedule.pm 1970-01-01 01:00:00.000000000 +0100 +++ new/MirrorCache-1.079/lib/MirrorCache/Task/ReportProjectSizeSchedule.pm 2024-07-24 16:14:18.000000000 +0200 @@ -0,0 +1,49 @@ +# Copyright (C) 2024 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +package MirrorCache::Task::ReportProjectSizeSchedule; +use Mojo::Base 'Mojolicious::Plugin'; +use DateTime; + +sub register { + my ($self, $app) = @_; + $app->minion->add_task(report_project_size_schedule => sub { _run($app, @_) }); +} + +my $HASHES_QUEUE = $ENV{MIRRORCACHE_HASHES_QUEUE} // 'hashes'; +my $DELAY = int($ENV{MIRRORCACHE_SCHEDULE_REPORT_PROJECT_SIZE_SCHEDULE_RETRY_INTERVAL} // 12 * 60 * 60); + +sub _run { + my ($app, $job, $once) = @_; + my $minion = $app->minion; + return $job->finish('Previous report job is still active') + unless my $guard = $minion->guard('report_project_size_schedule', 2*60*60); + + my $schema = $app->schema; + my @projects = $schema->resultset('Project')->all; + my $sunday = DateTime->today->day_of_week; + $sunday = 0 unless $sunday == 7; + for my $proj (@projects) { + my $path = $proj->path; + my $prio = $proj->prio // 1; + next if !$sunday && $prio < 1; + $minion->enqueue('report_project_size' => [ $proj->path ] => {priority => 15} => {queue => $HASHES_QUEUE}); + } + + return $job->finish if $once; + return $job->retry({delay => $DELAY}); +} + +1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm --- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm 2024-07-24 16:14:18.000000000 +0200 @@ -40,7 +40,7 @@ }, Project => { keys => [['id'], ['name'],], - cols => ['id', 'name', 'path', 'prio'], + cols => ['id', 'name', 'path', 'prio', 'size', 'file cnt', 'lm'], }, ); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Backstage.pm new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Backstage.pm --- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Backstage.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Backstage.pm 2024-07-24 16:14:18.000000000 +0200 @@ -46,6 +46,10 @@ if ($app->mcconfig->mirror_provider) { push @permanent_jobs, 'mirror_provider_sync'; } + eval { + my $projects = $app->mcproject->list; + push @permanent_jobs, 'report_project_size_schedule' if @$projects; + }; $app->plugin($_) for ( @@ -67,6 +71,8 @@ qw(MirrorCache::Task::ProjectSyncSchedule), qw(MirrorCache::Task::Cleanup), qw(MirrorCache::Task::Report), + qw(MirrorCache::Task::ReportProjectSize), + qw(MirrorCache::Task::ReportProjectSizeSchedule), qw(MirrorCache::Task::StatAggSchedule), ); } @@ -113,7 +119,7 @@ $app->hook( before_server_start => sub { - my $every = $ENV{MIRRORCACHE_PERMANENT_JOBS_CHECK_INTERVAL} // 15 * 60; + my $every = $ENV{MIRRORCACHE_PERMANENT_JOBS_CHECK_INTERVAL} // 5 * 60; $self->check_permanent_jobs; Mojo::IOLoop->next_tick( sub { @@ -132,6 +138,9 @@ eval { my $app = shift->app; my $minion = $app->minion; + my $guard = $minion->guard('check_permanent_jobs', 60); + return undef unless $guard; + my $jobs = $minion->jobs( { tasks => \@permanent_jobs, diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Dir.pm new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Dir.pm --- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2024-07-24 16:14:18.000000000 +0200 @@ -23,6 +23,8 @@ use Time::Piece; use Time::Seconds; use Time::ParseDate; +use Mojo::Date; + use Digest::Metalink; use Digest::Meta4; use MirrorCache::Utils; @@ -82,6 +84,7 @@ eval { $success = _render_hashes($dm) + || _render_if_not_modified($dm) || _render_small($dm) || _set_cache_control($dm) || _redirect_project_ln_geo($dm) @@ -745,6 +748,30 @@ return $c->render( 'dir', files => \@items, route => $dm->route, cur_path => $dir, folder_id => $id ); } +sub _render_if_not_modified { + my $dm = shift; + return undef if $root->is_remote; + my $c = $dm->c; + my $x = $c->req->headers->if_modified_since; + $c->log->error($c->dumper('DIR::render_if_not_modified1', $x, ref $x, scalar($x), ref scalar($x))) if $MCDEBUG; + return undef unless $x; + $x = scalar($x); + my $lms; + eval { + $lms = Mojo::Date->new($x)->epoch; + }; + $c->log->error($c->dumper('DIR::render_if_not_modified2', $lms)) if $MCDEBUG; + return undef unless int($lms // 0); + $dm->_init_path; + my ($path, undef) = $dm->path; + my $mtime = $root->is_file_mtime($path); + $c->log->error($c->dumper('DIR::render_if_not_modified3', $mtime)) if $MCDEBUG; + return undef unless int($mtime // 0); + return $c->render(text => '', status => 304) if $mtime <= $lms; + $c->log->error($c->dumper('DIR::render_if_not_modified4', $mtime, $lms)) if $MCDEBUG; + return undef; +} + sub _render_small { my $dm = shift; my $root_nfs = $mc_config->root_nfs; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Project.pm new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Project.pm --- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Project.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Project.pm 2024-07-24 16:14:18.000000000 +0200 @@ -54,11 +54,13 @@ eval { #the table may be missing - no big deal (only reports might be inaccurate if some other error occurred). eval { my @rows = $c->schema->resultset('Project')->search(undef, { order_by => { -desc => [qw/prio name/] } }); + my @projs; # we want to cache it, so move to simpler structure for my $r (@rows) { my %proj = ( id => $r->id, name => $r->name, path => $r->path, redirect => $r->redirect, prio => $r->prio ); - push @projects, \%proj; + push @projs, \%proj; } + @projects = @projs; $wasdberror = 0; $initialized = 1; }; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm --- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm 2024-07-24 16:14:18.000000000 +0200 @@ -119,7 +119,7 @@ $c->log->error($c->dumper('RENDER FOLDER REAL', $realfolder_id ? $realfolder_id : 'NULL')) if $MCDEBUG; my $fileoriginpath = $filepath; $fileoriginpath = $realdirname . '/' . $basename if $realdirname ne $dirname; - return $root->render_file($dm, $fileoriginpath, 1) if $dm->must_render_from_root; # && $root->is_reachable; + return $root->render_file($dm, $fileoriginpath, 1) if $dm->must_render_from_root && !$c->req->headers->if_modified_since; # && $root->is_reachable; if ($folder || $realfolder_id) { my $fldid = ($realfolder_id? $realfolder_id : $folder_id); @@ -152,6 +152,22 @@ $c->res->headers->vary('Accept, COUNTRY, X-COUNTRY, Fastly-SSL'); $c->res->headers->etag($dm->etag) if defined $dm->file_size; $c->res->headers->add('X-MEDIA-VERSION' => $dm->media_version) if $dm->media_version; + + my $mtime = $file->{mtime}; + if ($mtime) { # Check Last Modified Since header + my $lms; + eval { + if (my $x = $c->req->headers->if_modified_since) { + $c->log->error($c->dumper('RENDER IF MODIFIED SINCE1', $x)) if $MCDEBUG; + $lms = Mojo::Date->new($x)->epoch; + } + }; + $c->log->error($c->dumper('RENDER IF MODIFIED SINCE', $lms, $mtime)) if $MCDEBUG; + + # Not Modified + return $c->render(status => 304, text => '') if int($lms // 0) && int($mtime // 0) && $mtime <= $lms; + } + my $baseurl; # just hostname + eventual urldir (without folder and file) my $fullurl; # baseurl with path and filename if ($dm->metalink || $dm->meta4 || $dm->torrent || $dm->zsync || $dm->magnet) { @@ -844,6 +860,7 @@ return 1; } + $c->res->headers->content_disposition('attachment; filename="' .$filename. '.btih"'); $c->render(text => "$filename " . _calc_btih($filename, $file)); return 1; } @@ -862,6 +879,7 @@ $filename = Mojo::Util::url_escape($filename); $url = Mojo::Util::url_escape($url); + $c->res->headers->content_disposition('attachment; filename="' .$filename. '.magnet"'); $c->render(text => "magnet:?xt=urn:btih:$btih&xt=urn:md5:$md5&xl=$size&dn=$filename&as=$url"); return 1; } @@ -911,6 +929,7 @@ $footer = $footer . 'ee'; $c->res->headers->content_length(length($header) + length($pieces) + length($footer)); + $c->res->headers->content_disposition('attachment; filename="' .$filename. '.torrent"'); $c->write($header => sub () { $c->write($pieces => sub () { $c->write($footer => sub () {$c->finish}); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm --- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm 2024-07-24 16:14:18.000000000 +0200 @@ -78,6 +78,16 @@ return 0; } +sub is_file_mtime { + return 1 unless $_[1]; + for my $root (@roots) { + my $f = $root->[dir] . $root_subtree . $_[1]; + next unless -f $root->[dir] . $root_subtree . $_[1]; + return Mojo::File->new($f)->stat->mtime; + } + return 0; +} + sub is_dir { my ($self, $path) = @_; return 1 if !$path || $path eq '/'; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm --- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm 2024-07-24 16:14:18.000000000 +0200 @@ -67,7 +67,7 @@ my $c = shift; my $file = $c->param('file'); my $https = $c->param('https'); - return $c->render(status => 400) unless $file; + return $c->render(text => '', status => 400) unless $file; my $dm = MirrorCache::Datamodule->new->app($c->app); $dm->reset($c); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/resources/migrations/Pg.sql new/MirrorCache-1.079/lib/MirrorCache/resources/migrations/Pg.sql --- old/MirrorCache-1.077/lib/MirrorCache/resources/migrations/Pg.sql 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/resources/migrations/Pg.sql 2024-07-24 16:14:18.000000000 +0200 @@ -414,3 +414,8 @@ alter table folder_diff add column if not exists realfolder_id bigint; -- 36 up alter table project drop column etalon; +-- 37 up +alter table project + add column if not exists size bigint, + add column if not exists file_cnt bigint, + add column if not exists lm bigint; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/resources/migrations/mysql.sql new/MirrorCache-1.079/lib/MirrorCache/resources/migrations/mysql.sql --- old/MirrorCache-1.077/lib/MirrorCache/resources/migrations/mysql.sql 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/lib/MirrorCache/resources/migrations/mysql.sql 2024-07-24 16:14:18.000000000 +0200 @@ -426,3 +426,8 @@ alter table folder_diff add column if not exists realfolder_id bigint; -- 36 up alter table project drop constraint fk_project_etalon, drop column etalon; +-- 37 up +alter table project + add column if not exists size bigint, + add column if not exists file_cnt bigint, + add column if not exists lm bigint; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/t/environ/02-files-hashes.sh new/MirrorCache-1.079/t/environ/02-files-hashes.sh --- old/MirrorCache-1.077/t/environ/02-files-hashes.sh 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/t/environ/02-files-hashes.sh 2024-07-24 16:14:18.000000000 +0200 @@ -74,11 +74,11 @@ $mc/curl /download/folder1/file1.1.dat.meta4 | grep -o '<hash>5179db3d4263c9cb4ecf0edbc653ca460e3678b7</hash>' $mc/curl /download/folder1/file1.1.dat.metalink | grep -o '<hash piece="0">5179db3d4263c9cb4ecf0edbc653ca460e3678b7</hash>' -$mc/curl -I /download/folder1/file1.1.dat.btih | grep '200 OK' +$mc/curl -I /download/folder1/file1.1.dat.btih | grep -C20 '200 OK' | grep 'Content-Disposition: attachment; filename="file1.1.dat.btih"' $mc/curl /download/folder1/file1.1.dat.btih -$mc/curl -I /download/folder1/file1.1.dat.magnet | grep '200 OK' +$mc/curl -I /download/folder1/file1.1.dat.magnet | grep -C20 '200 OK' | grep 'Content-Disposition: attachment; filename="file1.1.dat.magnet"' $mc/curl /download/folder1/file1.1.dat.magnet -$mc/curl -I /download/folder1/file1.1.dat.torrent | grep '200 OK' +$mc/curl -I /download/folder1/file1.1.dat.torrent | grep -C20 '200 OK' | grep 'Content-Disposition: attachment; filename="file1.1.dat.torrent"' $mc/curl /download/folder1/file1.1.dat.torrent $mc/curl /download/folder1/file1.1.dat.metalink | xmllint --noout --format - diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/t/environ/02-files.sh new/MirrorCache-1.079/t/environ/02-files.sh --- old/MirrorCache-1.077/t/environ/02-files.sh 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/t/environ/02-files.sh 2024-07-24 16:14:18.000000000 +0200 @@ -6,9 +6,6 @@ MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0 $mc/gen_env MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL -$mc/start -$mc/status - ap8=$(environ ap8) ap7=$(environ ap7) @@ -35,10 +32,17 @@ done done +$mc/start +$mc/status + $ap7/start $ap8/start $mc/curl -I -H "Accept: */*, application/metalink+xml" /download/Folder1/repodata/repomd.xml | grep '200 OK' +$mc/curl -I -H "Accept: */*, application/metalink+xml" -H "If-Modified-Since: $(date -u --rfc-3339=seconds --date='1 second ago')" /download/Folder1/repodata/repomd.xml | grep '304 Not Modified' +$mc/curl -I -H "Accept: */*, application/metalink+xml" -H "If-Modified-Since: $(date -u --rfc-3339=seconds --date='1 hour ago')" /download/Folder1/repodata/repomd.xml | grep '200 OK' +$mc/curl -I -H "Accept: */*, application/metalink+xml" -H "If-Modified-Since: Sun, 06 Nov 1994 08:49:37 GMT" /download/Folder1/repodata/repomd.xml | grep '200 OK' +$mc/curl -I -H "Accept: */*, application/metalink+xml" -H "If-Modified-Since: Smoe 10 Garbage 10:53:46 UTC 2024x" /download/Folder1/repodata/repomd.xml | grep '200 OK' $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'" $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','ca','na'" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/t/environ/03-headquarter-subsidiaries-hashes.sh new/MirrorCache-1.079/t/environ/03-headquarter-subsidiaries-hashes.sh --- old/MirrorCache-1.077/t/environ/03-headquarter-subsidiaries-hashes.sh 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/t/environ/03-headquarter-subsidiaries-hashes.sh 2024-07-24 16:14:18.000000000 +0200 @@ -85,10 +85,8 @@ mc$i/sql_test file4.1.dat == "select hash.target from hash join file on id = file_id where name='file-Media.iso'" for x in '' metalink mirrorlist; do ext="" - par="" [ -z "$x" ] || ext=.$x - [ -z "$x" ] || par=?$x - mc$i/curl -I /folder1/file-Media.iso$ext | grep -C 10 302 | grep /folder1/file4.1.dat$par | grep -v /download/folder1/file4.1.dat + mc$i/curl -I /folder1/file-Media.iso$ext | grep -C 10 302 | grep /folder1/file4.1.dat$ext | grep -v /download/folder1/file4.1.dat done done @@ -115,24 +113,26 @@ sleep $DELAY mc9/curl -I /download/folder1/file-Media.iso | grep 'Location: /download/folder1/file4.1.dat' -mc9/curl -I /download/folder1/file-Media.iso.metalink | grep 'Location: /download/folder1/file4.1.dat?metalink=1' -mc9/curl -I /download/folder1/file-Media.iso.mirrorlist | grep 'Location: /download/folder1/file4.1.dat?mirrorlist=1' +mc9/curl -I /download/folder1/file-Media.iso.metalink | grep 'Location: /download/folder1/file4.1.dat\.metalink' +mc9/curl -I /download/folder1/file-Media.iso?metalink | grep 'Location: /download/folder1/file4.1.dat?metalink=' +mc9/curl -I /download/folder1/file-Media.iso.mirrorlist | grep 'Location: /download/folder1/file4.1.dat\.mirrorlist' +mc9/curl -I /download/folder1/file-Media.iso?mirrorlist | grep 'Location: /download/folder1/file4.1.dat?mirrorlist=' mc9/curl -I /download/folder1/xcurr.dat | grep "Location: http://$na_address/download/folder1/xcurr.dat" mc9/curl -I /download/folder1/xcurr.dat.metalink | grep "Location: http://$na_address/download/folder1/xcurr.dat.metalink" -mc9/curl -I /download/folder1/xcurr.dat?meta4 | grep "Location: http://$na_address/download/folder1/xcurr.dat.meta4" -mc9/curl -I /download/folder1/xcurr.dat.mirrorlist | grep '/folder1/file4.1.dat?mirrorlist=' +mc9/curl -I /download/folder1/xcurr.dat?meta4 | grep "Location: http://$na_address/download/folder1/xcurr.dat?meta4" +mc9/curl -I /download/folder1/xcurr.dat.mirrorlist | grep '/folder1/file4.1.dat.mirrorlist' -mc9/curl -I /download/folder1/xcurr.dat.zsync.mirrorlist | grep 'Location: /download/folder1/file4.1.dat.zsync?mirrorlist=' mc9/curl -I /download/folder1/xcurr.dat.zsync?mirrorlist | grep 'Location: /download/folder1/file4.1.dat.zsync?mirrorlist=' +mc9/curl -I /download/folder1/xcurr.dat.zsync.mirrorlist | grep 'Location: /download/folder1/file4.1.dat.zsync\.mirrorlist' mc6/sql_test file4.1.dat == "select target from file where name = 'xcurr.dat'" mc6/curl -IL /download/folder1/xcurr.dat | grep '200 OK' -mc6/curl -I /download/folder1/xcurr.dat.metalink | grep '/folder1/file4.1.dat?metalink=' +mc6/curl -I /download/folder1/xcurr.dat.metalink | grep '/folder1/file4.1.dat\.metalink' mc6/curl -I /download/folder1/xcurr.dat?meta4 | grep '/folder1/file4.1.dat?meta4=' -mc6/curl -I /download/folder1/xcurr.dat.mirrorlist | grep '/folder1/file4.1.dat?mirrorlist=' +mc6/curl -I /download/folder1/xcurr.dat.mirrorlist | grep '/folder1/file4.1.dat\.mirrorlist' # now the hashes on subsidiaries should be retried and match the headquarter for i in 6 7 8; do diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/t/environ/04-remote-mc.sh new/MirrorCache-1.079/t/environ/04-remote-mc.sh --- old/MirrorCache-1.077/t/environ/04-remote-mc.sh 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/t/environ/04-remote-mc.sh 2024-07-24 16:14:18.000000000 +0200 @@ -48,6 +48,7 @@ test 1 == $($mc/db/sql "select count(*) from folder_diff_file") $mc/curl -I /download/folder1/file2.1.dat | grep $($ng7/print_address) +$mc/curl -I -H "If-Modified-Since: $(date -u --rfc-3339=seconds)" /download/folder1/file2.1.dat | grep '304 Not Modified' mv $ng7/dt/folder1/file2.1.dat $ng8/dt/folder1/ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/t/environ/14-project-report.sh new/MirrorCache-1.079/t/environ/14-project-report.sh --- old/MirrorCache-1.077/t/environ/14-project-report.sh 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/t/environ/14-project-report.sh 2024-07-24 16:14:18.000000000 +0200 @@ -21,6 +21,8 @@ echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1 touch echo $x/dt/project1/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1 touch echo $x/dt/project2/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1 touch + echo 1234567 > $x/dt/project2/folder2/file1.1.dat + echo 1234 > $x/dt/project2/folder2/file2.1.dat done $ap3/start @@ -74,8 +76,13 @@ $mc/backstage/job mirror_probe_projects $mc/backstage/job -e report -a '["once"]' +$mc/backstage/job -e report_project_size_schedule -a '["once"]' $mc/backstage/shoot +$mc/sql_test 4 == "select file_cnt from project where path = '/project1'" +$mc/sql_test 2 == "select file_cnt from project where path = '/project2/folder1'" +$mc/sql_test 13 == "select size from project where path = '/project2/folder2'" + $mc/curl /report/mirrors | tidy --drop-empty-elements no | \ grep -A4 -F '<div class="repo">' | \ grep -A2 -F '"http://127.0.0.1:1304/project2/folder2">' | \ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.077/templates/app/project/index.html.ep new/MirrorCache-1.079/templates/app/project/index.html.ep --- old/MirrorCache-1.077/templates/app/project/index.html.ep 2024-06-20 07:43:31.000000000 +0200 +++ new/MirrorCache-1.079/templates/app/project/index.html.ep 2024-07-24 16:14:18.000000000 +0200 @@ -18,6 +18,9 @@ <th class="col_value">Name</th> <th class="col_value">Path</th> <th class="col_value">Prio</th> + <th class="col_ro">Size</th> + <th class="col_ro">File Count</th> + <th class="col_unixtime">Last Modified</th> <th class="col_action">Actions</th> </tr> </thead> ++++++ MirrorCache.obsinfo ++++++ --- /var/tmp/diff_new_pack.Zlvxh3/_old 2024-08-01 22:06:26.361440995 +0200 +++ /var/tmp/diff_new_pack.Zlvxh3/_new 2024-08-01 22:06:26.397442481 +0200 @@ -1,5 +1,5 @@ name: MirrorCache -version: 1.077 -mtime: 1718862211 -commit: 8e27b8285ca98c3110c0cc8c1438fe4a88157fce +version: 1.079 +mtime: 1721830458 +commit: 1fd848f467a79f3c2697593716894e3ee3ea7238