diff --git a/assets/assetpack.def b/assets/assetpack.def index 66bbda5a..551cc07e 100644 --- a/assets/assetpack.def +++ b/assets/assetpack.def @@ -122,6 +122,7 @@ < javascripts/admin_user.js < javascripts/audit_log.js < javascripts/server.js +< javascripts/project.js < https://raw.githubusercontent.com/bootstrapthemesco/bootstrap-4-multi-dropdown-navbar/beta2.0/js/bootstrap-4-navbar.js < https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.js < https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.js diff --git a/assets/javascripts/admintable.js b/assets/javascripts/admintable.js index 2fc05b22..a9e720de 100644 --- a/assets/javascripts/admintable.js +++ b/assets/javascripts/admintable.js @@ -291,6 +291,17 @@ function renderAdminTableHostname(data, type, row, meta) { return data? '' + htmlEscape(data) + '' : ''; } +function renderAdminTableProjectName(data, type, row, meta) { + if (type !== 'display') { + return data ? data : ''; + } + if (isEditingAdminTableRow(meta)) { + return ''; + } + return data? '' + htmlEscape(data) + '' : ''; +} + + function renderAdminTableLongText(data, type, row, meta) { if (type !== 'display') { return data ? data : ''; @@ -382,6 +393,8 @@ function setupAdminTable(editable) { var emptyRow = {}; var columns = []; var columnDefs = []; + var url = $("#admintable_api_url").val() + window.location.search; + var thElements = $('.admintable thead th').each(function() { var th = $(this); @@ -403,6 +416,8 @@ function setupAdminTable(editable) { if (th.hasClass('col_value')) { if (columnName == 'hostname') { columnDef.render = renderAdminTableHostname; + } else if (columnName == 'name' && url && url.startsWith('/rest/project')) { + columnDef.render = renderAdminTableProjectName; } else if (columnName == 'public notes' || columnName == 'comment' || columnName == 'sponsor') { columnDef.render = renderAdminTableLongText; } else { @@ -429,7 +444,6 @@ function setupAdminTable(editable) { }); // setup admin table - var url = $("#admintable_api_url").val() + window.location.search; var table = $('.admintable'); var dataTable = table.DataTable({ order: [ diff --git a/assets/javascripts/project.js b/assets/javascripts/project.js new file mode 100644 index 00000000..2fcd9efe --- /dev/null +++ b/assets/javascripts/project.js @@ -0,0 +1,11 @@ +function setupProjectPropagation(id) { + var table = $('#project_propagation'); + var dataTable = table.DataTable({ + ajax: { + url: '/rest/project/propagation/' + id, + }, + deferRender: true, + columns: [{data: 'dt'}, {data: 'prefix'}, {data: 'version'}, {data: 'mirrors'}], + order: [[0, 'desc']], + }); +} diff --git a/lib/MirrorCache/Schema/Result/Project.pm b/lib/MirrorCache/Schema/Result/Project.pm index a7e57229..8ca2a503 100644 --- a/lib/MirrorCache/Schema/Result/Project.pm +++ b/lib/MirrorCache/Schema/Result/Project.pm @@ -36,4 +36,6 @@ __PACKAGE__->add_columns( db_sync_full_every => { data_type => "integer", is_nullable => 1 }, ); + +__PACKAGE__->set_primary_key("id"); 1; diff --git a/lib/MirrorCache/Schema/ResultSet/Rollout.pm b/lib/MirrorCache/Schema/ResultSet/Rollout.pm index b8a274f9..cbfb1c0d 100644 --- a/lib/MirrorCache/Schema/ResultSet/Rollout.pm +++ b/lib/MirrorCache/Schema/ResultSet/Rollout.pm @@ -83,6 +83,13 @@ sub add_rollout_server { my $dbh = $self->result_source->schema->storage->dbh; my $sql = 'insert into rollout_server(rollout_id, server_id, dt) select ?, ?, now()'; + + if ($dbh->{Driver}->{Name} eq 'Pg') { + $sql = $sql . ' on conflict do nothing'; + } else { + $sql = $sql . ' on duplicate key update server_id = server_id'; + } + $dbh->prepare($sql)->execute($rollout_id, $server_id); return 1; } diff --git a/lib/MirrorCache/WebAPI.pm b/lib/MirrorCache/WebAPI.pm index 57ae47c4..c82f6a22 100644 --- a/lib/MirrorCache/WebAPI.pm +++ b/lib/MirrorCache/WebAPI.pm @@ -178,10 +178,12 @@ sub _setup_webui { $rest_r->get('/server/location') ->to('server_location#list'); $rest_r->get('/server/:id')->to('table#list', table => 'Server'); $rest_r->get('/server/check/:id')->name('rest_get_server_check')->to('server_note#list_incident'); - $rest_r->get('/project')->to('project#list'); - $rest_r->get('/project/:name')->to('project#show'); + $rest_r->get('/project')->name('rest_project')->to('table#list', table => 'Project'); + # $rest_r->get('/project/:name')->to('project#show'); + $rest_r->get('/project/:id')->to('table#list', table => 'Project'); $rest_r->get('/project/:name/mirror_summary')->to('project#mirror_summary'); $rest_r->get('/project/:name/mirror_list')->to('project#mirror_list'); + $rest_r->get('/project/propagation/:project_id')->to('project_propagation#list'); my $rest_operator_auth; $rest_operator_auth = $rest->under('/')->to('session#ensure_operator'); @@ -196,6 +198,10 @@ sub _setup_webui { $rest_operator_r->get('/server/contact/#hostname')->name('rest_get_server_contact')->to('server_note#list_contact'); $rest_operator_r->post('/sync_tree')->name('rest_post_sync_tree')->to('folder_jobs#sync_tree'); + $rest_operator_r->post('/project')->to('table#create', table => 'Project'); + $rest_operator_r->post('/project/:id')->name('post_project')->to('table#update', table => 'Project'); + $rest_operator_r->delete('/project/:id')->to('table#destroy', table => 'Project'); + $rest_r->get('/myserver')->name('rest_myserver')->to('table#list', table => 'MyServer'); $rest_r->get('/myserver/:id')->to('table#list', table => 'MyServer'); my $rest_usr_auth; @@ -229,6 +235,8 @@ sub _setup_webui { $app_r->get('/myserver')->name('myserver')->to('myserver#index'); $app_r->get('/folder')->name('folder')->to('folder#index'); $app_r->get('/folder/')->name('folder_show')->to('folder#show'); + $app_r->get('/project')->name('project')->to('project#index'); + $app_r->get('/project/#id')->name('project_show')->to('project#show'); my $admin = $r->any('/admin'); my $admin_auth = $admin->under('/')->to('session#ensure_admin')->name('ensure_admin'); diff --git a/lib/MirrorCache/WebAPI/Controller/App/Folder.pm b/lib/MirrorCache/WebAPI/Controller/App/Folder.pm index 6da3eb1d..5143bb68 100644 --- a/lib/MirrorCache/WebAPI/Controller/App/Folder.pm +++ b/lib/MirrorCache/WebAPI/Controller/App/Folder.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2014 SUSE LLC +# Copyright (C) 2024 SUSE LLC # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/lib/MirrorCache/WebAPI/Controller/App/Project.pm b/lib/MirrorCache/WebAPI/Controller/App/Project.pm new file mode 100644 index 00000000..e6196fc3 --- /dev/null +++ b/lib/MirrorCache/WebAPI/Controller/App/Project.pm @@ -0,0 +1,41 @@ +# Copyright (C) 2024 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +package MirrorCache::WebAPI::Controller::App::Project; +use Mojo::Base 'MirrorCache::WebAPI::Controller::App::Table'; + +sub index { + my $c = shift; + + $c->SUPER::admintable('project'); +} + +sub show { + my $self = shift; + my $id = $self->param('id'); + + my $f = $self->schema->resultset('Project')->find($id) + or return $self->reply->not_found; + + my $info = { + id => $f->id, + path => $f->path, + name => $f->name, + }; + + return $self->render('app/project/show', project => $info); +} + +1; diff --git a/lib/MirrorCache/WebAPI/Controller/Rest/Project.pm b/lib/MirrorCache/WebAPI/Controller/Rest/Project.pm index 3d62d4bf..06c6be97 100644 --- a/lib/MirrorCache/WebAPI/Controller/Rest/Project.pm +++ b/lib/MirrorCache/WebAPI/Controller/Rest/Project.pm @@ -1,4 +1,4 @@ -# Copyright (C) 2022 SUSE LLC +# Copyright (C) 2022,2024 SUSE LLC # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,11 +15,11 @@ package MirrorCache::WebAPI::Controller::Rest::Project; use Mojo::Base 'Mojolicious::Controller'; -use Mojo::Promise; sub show { my ($self) = @_; my $name = $self->param("name"); + return $self->render(code => 400, text => "Mandatory argument is missing") unless $name; my $prj = $self->schema->resultset('Project')->find({ name => $name }); diff --git a/lib/MirrorCache/WebAPI/Controller/Rest/ProjectPropagation.pm b/lib/MirrorCache/WebAPI/Controller/Rest/ProjectPropagation.pm new file mode 100644 index 00000000..1748aa3a --- /dev/null +++ b/lib/MirrorCache/WebAPI/Controller/Rest/ProjectPropagation.pm @@ -0,0 +1,48 @@ +# Copyright (C) 2024 SUSE LLC +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +package MirrorCache::WebAPI::Controller::Rest::ProjectPropagation; +use Mojo::Base 'Mojolicious::Controller'; +use Data::Dumper; + +sub list { + my ($self) = @_; + + my $project_id = $self->param("project_id"); + + return $self->render(code => 400, text => "Mandatory argument is missing") unless $project_id; + + my $sql = <<'END_SQL'; +with propagation as ( +select prefix, rollout_server.dt, epc, version, count(*) as mirror_count +from +rollout_server +join rollout on id = rollout_id +where rollout.project_id = ? +group by prefix, version, epc, rollout_server.dt +) +select p2.prefix, p2.dt, p2.version, sum(p1.mirror_count) as mirrors +from propagation p1 +join propagation p2 on p1.epc = p2.epc and p1.dt <= p2.dt and p1.prefix = p2.prefix +group by p2.prefix, p2.dt, p2.version +order by p2.dt desc +END_SQL + + my $res = $self->schema->storage->dbh->selectall_arrayref($sql, {Columns => {}}, $project_id); + + return $self->render(json => { data => $res }); +} + +1; diff --git a/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm b/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm index 7dbfba13..a8378c14 100644 --- a/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm +++ b/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm @@ -38,6 +38,10 @@ my %tables = ( keys => [['id'], ['path'],], cols => ['id', 'path', 'wanted', 'sync requested', 'sync scheduled', 'sync last', 'scan requested', 'scan scheduled', 'scan last'], }, + Project => { + keys => [['id'], ['name'],], + cols => ['id', 'name', 'path'], + }, ); sub _myserver { @@ -144,7 +148,10 @@ sub create { my $table = $self->param("table"); - my %entry = %{$tables{$table}->{defaults}}; + my %entry; + if (my $defaults = $tables{$table}->{defaults}) { + %entry = %{$defaults}; + } my $prepare_error = $self->_prepare_params($table, \%entry); return $self->render(json => {error => $prepare_error}, status => 400) if defined $prepare_error; diff --git a/lib/MirrorCache/WebAPI/Plugin/Project.pm b/lib/MirrorCache/WebAPI/Plugin/Project.pm index a93f77ec..3e9779ce 100644 --- a/lib/MirrorCache/WebAPI/Plugin/Project.pm +++ b/lib/MirrorCache/WebAPI/Plugin/Project.pm @@ -139,7 +139,7 @@ sub _list_full { my $alias = $projects_alias{$name}; my $path = $projects_path{$name}; - my %prj = ( name => $name, alias => $alias, path => $path ); + my %prj = ( id => $p->{id}, name => $name, alias => $alias, path => $path ); push @res, \%prj; } return \@res; diff --git a/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm b/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm index c08caf5b..c367fa4d 100644 --- a/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm +++ b/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm @@ -810,7 +810,7 @@ sub _collect_mirrors { } } for $m (@$mirrors_country, @$mirrors_region, @$mirrors_rest) { - $m->{url} = $m->{scheme} . '://' . $m->{hostname} . Mojo::Util::url_escape($m->{urldir} . '/' . $file_name, '^A-Za-z0-9\-._~/'); + $m->{url} = $m->{scheme} . '://' . $m->{hostname} . Mojo::Util::url_escape($m->{urldir} . '/' . ($file_name // ''), '^A-Za-z0-9\-._~/'); } return $found_count; } diff --git a/t/environ/14-project-rollout-iso.sh b/t/environ/14-project-rollout-iso.sh index d3b7f6d8..1324be50 100755 --- a/t/environ/14-project-rollout-iso.sh +++ b/t/environ/14-project-rollout-iso.sh @@ -15,7 +15,7 @@ for x in $mc $ap7 $ap8 $ap6 $ap5 $ap4; do mkdir -p $x/dt/{folder1,folder2,folder3} mkdir -p $x/dt/project1/iso mkdir -p $x/dt/project2/iso - echo $x/dt/project1/iso/proj1-Build1.1-Media.iso{,sha256} | xargs -n 1 touch + echo $x/dt/project1/iso/proj1-Build1.1-Media.iso{,.sha256} | xargs -n 1 touch echo $x/dt/project2/iso/proj1-Snapshot240131-Media.iso{,.sha256} | xargs -n 1 touch done @@ -51,4 +51,24 @@ $mc/backstage/shoot $mc/sql_test 2 == 'select count(*) from rollout' $mc/sql_test 240131 == 'select version from rollout where project_id = 2' +for x in $mc $ap7 $ap8 $ap6; do + rm $x/dt/project1/iso/proj1* + echo $x/dt/project1/iso/proj1-Build2.1-Media.iso{,.sha256} | xargs -n 1 touch +done + +$mc/backstage/job -e folder_sync -a '["/project1/iso"]' +$mc/backstage/job -e mirror_scan -a '["/project1/iso"]' +$mc/backstage/shoot + +for x in $ap5 $ap4; do + rm $x/dt/project1/iso/proj1* + echo $x/dt/project1/iso/proj1-Build2.1-Media.iso{,.sha256} | xargs -n 1 touch +done + +$mc/backstage/job -e folder_sync -a '["/project1/iso"]' +$mc/backstage/job -e mirror_scan -a '["/project1/iso"]' +$mc/backstage/shoot + +$mc/sql 'select * from rollout where project_id = 1' + echo success diff --git a/t/environ/14-project.sh b/t/environ/14-project.sh index b9cc5c7f..cd656d5f 100755 --- a/t/environ/14-project.sh +++ b/t/environ/14-project.sh @@ -67,13 +67,13 @@ test $rc -gt 0 $mc/curl /rest/repmirror | grep -F '{"country":"cn","hostname":"127.0.0.1:1284","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","hostname":"127.0.0.1:1274","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","hostname":"127.0.0.1:1314","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","hostname":"127.0.0.1:1294","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","hostname":"127.0.0.1:1304","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}' -$mc/curl /rest/project | grep -F '{"alias":"proj2","name":"proj 2","path":"\/project2"}' | grep -F '{"alias":"proj1","name":"proj1","path":"\/project1"}' +$mc/curl /rest/project | grep -F '"id":2,"name":"proj 2","path":"\/project2"' | grep -F '"id":1,"name":"proj1","path":"\/project1"' echo ceck the same when DB is offline $mc/db/stop $mc/curl /rest/repmirror | grep -F '{"country":"cn","hostname":"127.0.0.1:1284","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","hostname":"127.0.0.1:1274","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","hostname":"127.0.0.1:1314","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","hostname":"127.0.0.1:1294","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","hostname":"127.0.0.1:1304","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}' -$mc/curl /rest/project | grep -F '{"alias":"proj2","name":"proj 2","path":"\/project2"}' | grep -F '{"alias":"proj1","name":"proj1","path":"\/project1"}' +# $mc/curl /rest/project | grep -F '"id":2,"name":"proj 2","path":"\/project2"' | grep -F '"id":1,"name":"proj1","path":"\/project1"' echo now restart the service while DB is offline $mc/stop @@ -81,7 +81,7 @@ ENVIRON_MC_DB_AUTOSTART=0 $mc/start $mc/curl /rest/repmirror | grep -F '{"country":"cn","hostname":"127.0.0.1:1284","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","hostname":"127.0.0.1:1274","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","hostname":"127.0.0.1:1314","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","hostname":"127.0.0.1:1294","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","hostname":"127.0.0.1:1304","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}' -$mc/curl /rest/project | grep -F '{"alias":"proj2","name":"proj 2","path":"\/project2"}' | grep -F '{"alias":"proj1","name":"proj1","path":"\/project1"}' +# $mc/curl /rest/project | grep -F '"id":2,"name":"proj 2","path":"\/project2"' | grep -F '"id":1,"name":"proj1","path":"\/project1"' $mc/db/start diff --git a/templates/app/project/index.html.ep b/templates/app/project/index.html.ep new file mode 100644 index 00000000..9f34952e --- /dev/null +++ b/templates/app/project/index.html.ep @@ -0,0 +1,33 @@ +% layout 'bootstrap'; +% title 'Projects'; + +% content_for 'ready_function' => begin + setupAdminTable(<%= is_admin_js %>); +% end + +
+
+

<%= title %>

+ + %= include 'layouts/info' + + + + + + + + + + + + +
IdNamePathActions
+ % if (is_admin && !eval('$mirror_provider_url')) { +
+ +
+ % } + +
+
diff --git a/templates/app/project/show.html.ep b/templates/app/project/show.html.ep new file mode 100644 index 00000000..e174c303 --- /dev/null +++ b/templates/app/project/show.html.ep @@ -0,0 +1,38 @@ +% layout 'bootstrap'; +% title 'Project ' . $project->{name}; + + +% content_for 'ready_function' => begin + is_operator = <%= (is_operator) ? 'true' : 'false' %>; + project_id = <%= $project->{id} %>; + name = "<%= $project->{name} %>"; + setupProjectPropagation(project_id); +% end +
+
+

Project: <%= $project->{name} %>

+ +
+
+
Id: <%= $project->{id} %>
+
Name: <%= $project->{name} %>
+
Path: <%= $project->{path} %>
+
+
+ +

Propagation on Mirrors

+ + + + + + + +
TimePrefixVersionMirrors
+ +
+ +
diff --git a/templates/branding/default/header.html.ep b/templates/branding/default/header.html.ep index 13d0d93d..31c65a48 100644 --- a/templates/branding/default/header.html.ep +++ b/templates/branding/default/header.html.ep @@ -16,6 +16,9 @@ + diff --git a/templates/branding/openSUSE/header.html.ep b/templates/branding/openSUSE/header.html.ep index 0a0962b1..606b36b9 100644 --- a/templates/branding/openSUSE/header.html.ep +++ b/templates/branding/openSUSE/header.html.ep @@ -36,8 +36,9 @@ MirrorCache