From fe9fda3eacf2591ba7ecad8f55df970e5da05128 Mon Sep 17 00:00:00 2001 From: Shree Vatsa N Date: Mon, 9 Jan 2023 13:14:43 +0530 Subject: [PATCH 1/4] Add support for volume rebalance with subcommand 'volume rebalance start/stop' Signed-off-by: Shree Vatsa N --- mgr/src/cmds/volumes.cr | 50 +++++++++++++ mgr/src/server/plugins/volume_expand.cr | 6 +- .../plugins/volume_rebalance_start_stop.cr | 73 +++++++++++++++++++ mgr/src/server/plugins/volume_utils.cr | 59 ++++++++++++++- mgr/src/server/services/fix_layout.cr | 2 +- mgr/src/server/services/migrate_data.cr | 22 ++++++ sdk/crystal/src/volumes.cr | 19 +++++ 7 files changed, 226 insertions(+), 5 deletions(-) create mode 100644 mgr/src/server/plugins/volume_rebalance_start_stop.cr create mode 100644 mgr/src/server/services/migrate_data.cr diff --git a/mgr/src/cmds/volumes.cr b/mgr/src/cmds/volumes.cr index 285e5505..24709d2b 100644 --- a/mgr/src/cmds/volumes.cr +++ b/mgr/src/cmds/volumes.cr @@ -336,6 +336,10 @@ handler "volume.expand" do |args| puts "Volume #{req.name} expanded successfully" puts "ID: #{volume.id}" + + puts "Proceed to the rebalancing of volume #{req.name} by following the below steps." + puts "To start the rebalancing of volume: `kadalu volume rebalnce-start #{args.pool_name}/#{req.name}`" + puts "To force stop the rebalancing of volume: `kadalu volume rebalnce-stop #{args.pool_name}/#{req.name}`" end rescue ex : InvalidVolumeRequest STDERR.puts "Volume expand failed" @@ -343,3 +347,49 @@ handler "volume.expand" do |args| exit 1 end end + +command "volume.rebalance-start", "Start Rebalancing Kadalu Storage volume" do |parser, _| + parser.banner = "Usage: kadalu volume rebalance-start POOL/VOLNAME [arguments]" +end + +handler "volume.rebalance-start" do |args| + begin + command_error "Pool/Volname is required" if args.pos_args.size == 0 + args.pool_name, volume_name = pool_and_volume_name(args.pos_args.size > 0 ? args.pos_args[0] : "") + api_call(args, "Failed to start rebalancing of volume") do |client| + volume = client.pool(args.pool_name).volume(volume_name).rebalance_start + + handle_json_output(volume, args) + + puts "Rebalance of Volume #{volume.name} started" + end + rescue ex : InvalidVolumeRequest + STDERR.puts "Starting of volume rebalance failed" + STDERR.puts ex + exit 1 + end +end + +command "volume.rebalance-stop", "Stop Rebalancing Kadalu Storage volume" do |parser, _| + parser.banner = "Usage: kadalu volume rebalance-stop POOL/VOLNAME [arguments]" +end + +handler "volume.rebalance-stop" do |args| + begin + command_error "Pool/Volname is required" if args.pos_args.size == 0 + args.pool_name, volume_name = pool_and_volume_name(args.pos_args.size > 0 ? args.pos_args[0] : "") + next unless (args.script_mode || yes("Are you sure you want to stop rebalancing of volume? [y/N]")) + + api_call(args, "Failed to stop rebalancing of volume.") do |client| + volume = client.pool(args.pool_name).volume(volume_name).rebalance_stop + + handle_json_output(volume, args) + + puts "Rebalancing of Volume #{volume.name} stopped" + end + rescue ex : InvalidVolumeRequest + STDERR.puts "Stopping of volume rebalance failed" + STDERR.puts ex + exit 1 + end +end diff --git a/mgr/src/server/plugins/volume_expand.cr b/mgr/src/server/plugins/volume_expand.cr index c9350572..7e5e24c4 100644 --- a/mgr/src/server/plugins/volume_expand.cr +++ b/mgr/src/server/plugins/volume_expand.cr @@ -151,6 +151,10 @@ put "/api/v1/pools/:pool_name/volumes" do |env| existing_nodes = participating_nodes(pool_name, volume) + # Add only the first existing node for fix-layout service + services = add_fix_layout_service(services, pool.not_nil!.name, req.name, existing_nodes[0], + volume.not_nil!.distribute_groups[0].storage_units[0]) + # Remove duplicated node objects to avoid multiple node_actions to same node. all_unique_nodes = (existing_nodes + nodes).uniq(&.id) @@ -163,7 +167,7 @@ put "/api/v1/pools/:pool_name/volumes" do |env| ACTION_MANAGE_SERVICES, pool_name, all_unique_nodes, - {services, volfiles, rollback_volume}.to_json + {services, volfiles, rollback_volume, "start"}.to_json ) api_exception(!resp.ok, node_errors("Failed to restart SHD/start fix-layout service", resp.node_responses).to_json) diff --git a/mgr/src/server/plugins/volume_rebalance_start_stop.cr b/mgr/src/server/plugins/volume_rebalance_start_stop.cr new file mode 100644 index 00000000..379a13b7 --- /dev/null +++ b/mgr/src/server/plugins/volume_rebalance_start_stop.cr @@ -0,0 +1,73 @@ +require "moana_types" + +require "../conf" +require "./helpers" +require "../datastore/*" +require "./ping" +require "./volume_utils.cr" + +def rebalance_start_stop(env, action) + pool_name = env.params.url["pool_name"] + volume_name = env.params.url["volume_name"] + + forbidden_api_exception(!Datastore.maintainer?(env.user_id, pool_name, volume_name)) + + volume = Datastore.get_volume(pool_name, volume_name) + api_exception(volume.nil?, {"error": "Volume doesn't exists"}.to_json) + volume = volume.not_nil! + pool = volume.not_nil!.pool + + nodes = participating_nodes(pool_name, volume) + + # TODO: Add to missed_ops if a node is not reachable [Check if this is required, since node ping check is done] + + # Validate if all the nodes are reachable. + resp = dispatch_action(ACTION_PING, pool_name, nodes, "") + api_exception(!resp.ok, node_errors("Not all participant nodes are reachable", resp.node_responses).to_json) + + # Generate Services and Volfiles if Volume to be started + services, volfiles = services_and_volfiles(volume) + + # Node list where migrate data process is to be run. + migrate_data_nodes = [] of String + + # Add node of first storage_unit of every distribute group + volume.not_nil!.distribute_groups.each do |dist_grp| + services = add_migrate_data_service(services, pool.not_nil!.name, volume.not_nil!.name, + dist_grp.storage_units[0].node, dist_grp.storage_units[0]) + migrate_data_nodes.push(dist_grp.storage_units[0].node.name) + end + + resp = dispatch_action( + ACTION_MANAGE_SERVICES, + pool_name, + Datastore.get_nodes(pool_name, migrate_data_nodes.uniq), + {services, volfiles, volume, action}.to_json + ) + + api_exception(!resp.ok, node_errors("Failed to #{action} rebalancing of Volume", resp.node_responses).to_json) + + # Save Services details + services.each do |node_id, svcs| + svcs.each do |svc| + if action == "start" + # Enable each Services + Datastore.enable_service(pool.id, node_id, svc) + else + # Disable each Services + Datastore.disable_service(pool.id, node_id, svc) + end + end + end + + env.response.status_code = 200 + volume.to_json +end + +post "/api/v1/pools/:pool_name/volumes/:volume_name/rebalance_start" do |env| + rebalance_start_stop(env, "start") +end + +post "/api/v1/pools/:pool_name/volumes/:volume_name/rebalance_stop" do |env| + rebalance_start_stop(env, "stop") +end diff --git a/mgr/src/server/plugins/volume_utils.cr b/mgr/src/server/plugins/volume_utils.cr index 18d10fa0..9ab9b91f 100644 --- a/mgr/src/server/plugins/volume_utils.cr +++ b/mgr/src/server/plugins/volume_utils.cr @@ -14,11 +14,13 @@ TEST_XATTR_VALUE = "testvalue" VOLUME_ID_XATTR_NAME = "trusted.glusterfs.volume-id" alias VolumeRequestToNode = Tuple(Hash(String, Array(MoanaTypes::ServiceUnit)), Hash(String, Array(MoanaTypes::Volfile)), MoanaTypes::Volume) +alias VolumeRequestToNodeWithAction = Tuple(Hash(String, Array(MoanaTypes::ServiceUnit)), Hash(String, Array(MoanaTypes::Volfile)), MoanaTypes::Volume, String) alias ServiceRequestToNode = Tuple(Hash(String, Array(MoanaTypes::ServiceUnit))) ACTION_VALIDATE_VOLUME_CREATE = "validate_volume_create" ACTION_VOLUME_CREATE = "volume_create" ACTION_VOLUME_CREATE_STOPPED = "volume_create_stopped" +ACTION_MANAGE_SERVICES = "manage_services" node_action ACTION_VALIDATE_VOLUME_CREATE do |data, _env| req = MoanaTypes::Volume.from_json(data) @@ -33,6 +35,13 @@ node_action ACTION_VOLUME_CREATE_STOPPED do |data, _env| handle_volume_create(data, stopped: true) end +node_action ACTION_MANAGE_SERVICES do |data, _env| + services, volfiles, rollback_volume, action = VolumeRequestToNodeWithAction.from_json(data) + save_volfiles(volfiles) + sighup_processes(services) + restart_shd_service_and_manage_rebalance_services(services, action) +end + def volfile_get(name) # TODO: Add logic to read from the Templates directory case name @@ -129,14 +138,21 @@ def validate_volume_create(req) NodeResponse.new(true, "") end -def restart_shd_service_and_start_fix_layout_service(services) +def restart_shd_service_and_manage_rebalance_services(services, action = "start") unless services[GlobalConfig.local_node.id]?.nil? services[GlobalConfig.local_node.id].each do |service| svc = Service.from_json(service.to_json) if svc.name == "shdservice" svc.restart - elsif svc.name == "fixlayoutservice" - svc.start + elsif svc.name == "fixlayoutservice" || svc.name == "migratedataservice" + status_file_path = "/var/lib/kadalu/#{svc.id.gsub("/", "%2F")}.json" + FileUtils.rm(status_file_path) if File.exists?(status_file_path) + if action == "start" + svc.start + else + svc.stop + end + end end end @@ -527,3 +543,40 @@ def add_fix_layout_service(services, pool_name, volume_name, node, storage_unit) services end + +def add_migrate_data_service(services, pool_name, volume_name, node, storage_unit) + service = MigrateDataService.new(pool_name, volume_name, storage_unit) + services[node.id] << service.unit + + services +end + +def handle_volume_rebalance_start_stop(data, action) + services, volfiles, _ = VolumeRequestToNode.from_json(data) + + if action == "start" && !volfiles[GlobalConfig.local_node.id]?.nil? + Dir.mkdir_p(Path.new(GlobalConfig.workdir, "volfiles")) + volfiles[GlobalConfig.local_node.id].each do |volfile| + File.write(Path.new(GlobalConfig.workdir, "volfiles", "#{volfile.name}.vol"), volfile.content) + end + end + + unless services[GlobalConfig.local_node.id]?.nil? + # TODO: Hard coded path change? + Dir.mkdir_p("/var/log/kadalu") + Dir.mkdir_p("/run/kadalu") + services[GlobalConfig.local_node.id].each do |service| + svc = Service.from_json(service.to_json) + if svc.name == "migratedataservice" + if action == "start" + svc.start + else + svc.stop + end + end + end + end + + NodeResponse.new(true, "") +end + diff --git a/mgr/src/server/services/fix_layout.cr b/mgr/src/server/services/fix_layout.cr index 97d520e4..e7663be8 100644 --- a/mgr/src/server/services/fix_layout.cr +++ b/mgr/src/server/services/fix_layout.cr @@ -10,7 +10,7 @@ class FixLayoutService < Service @create_pid_file = true @path = Path[PROGRAM_NAME].expand.to_s - @id = "#{volume_name}-fix-layout" + @id = "rebalance-fix-layout-#{storage_unit.path}" @pid_file = "/run/kadalu/#{@id}.pid" @args = [ "_rebalance", "--fix-layout", diff --git a/mgr/src/server/services/migrate_data.cr b/mgr/src/server/services/migrate_data.cr new file mode 100644 index 00000000..3b6d14c9 --- /dev/null +++ b/mgr/src/server/services/migrate_data.cr @@ -0,0 +1,22 @@ +require "./services" + +class MigrateDataService < Service + getter path : String, + args : Array(String), + pid_file : String, + id : String + + def initialize(pool_name, volume_name, storage_unit) + @create_pid_file = true + + @path = Path[PROGRAM_NAME].expand.to_s + @id = "rebalance-migrate-data-#{storage_unit.path}" + @pid_file = "/run/kadalu/#{@id}.pid" + @args = [ + "_rebalance", "--migrate-data", + "#{pool_name}/#{volume_name}", + storage_unit.path, "--volfile-servers", + "#{storage_unit.node.name}:#{storage_unit.port}", + ] + end +end diff --git a/sdk/crystal/src/volumes.cr b/sdk/crystal/src/volumes.cr index 79ec63d2..ad61109a 100644 --- a/sdk/crystal/src/volumes.cr +++ b/sdk/crystal/src/volumes.cr @@ -202,5 +202,24 @@ module StorageManager StorageManager.error_response(response) end end + + def volume_rebalance_start_stop(action) + url = "#{@client.url}/api/v1/pools/#{@pool_name}/volumes/#{@name}/rebalance_#{action}" + + response = StorageManager.http_post(url, "{}", headers: @client.auth_header) + if response.status_code == 200 + MoanaTypes::Volume.from_json(response.body) + else + StorageManager.error_response(response) + end + end + + def rebalance_start + volume_rebalance_start_stop("start") + end + + def rebalance_stop + volume_rebalance_start_stop("stop") + end end end From e0960d236d591830ffa0d01a89ad82044d3cefb1 Mon Sep 17 00:00:00 2001 From: Shree Vatsa N Date: Wed, 11 Jan 2023 13:17:56 +0530 Subject: [PATCH 2/4] address review 2 Signed-off-by: Shree Vatsa N --- mgr/src/cmds/rebalance_process.cr | 50 +++++++++++++++++++++---- mgr/src/server/services/fix_layout.cr | 2 +- mgr/src/server/services/migrate_data.cr | 2 +- tests/all/volumes.t | 7 +++- 4 files changed, 50 insertions(+), 11 deletions(-) diff --git a/mgr/src/cmds/rebalance_process.cr b/mgr/src/cmds/rebalance_process.cr index 3ff7853d..bc0a9f1f 100644 --- a/mgr/src/cmds/rebalance_process.cr +++ b/mgr/src/cmds/rebalance_process.cr @@ -139,6 +139,7 @@ class Rebalancer next if @ignore_paths.includes?(rel_path.to_s) backend_full_path = Path.new(@backend_dir, rel_path) + # puts "backend_full_path: #{backend_full_path}" if File.directory?(backend_full_path) # TODO: Fix ENOENT issue in mount and remove list entries for, # Correct use of rebalance multiprocesses & improved time complexity. @@ -147,9 +148,7 @@ class Rebalancer all_dirs << rel_path next end - mnt_full_path = Path.new(@mount_dir, rel_path) - # Stat the file from the @backend_dir to check the size begin file_info = File.info(backend_full_path, follow_symlinks: false) @@ -161,17 +160,52 @@ class Rebalancer next end + + # puts "@mount_dir: #{@mount_dir}" + # puts "rel_path: #{rel_path}" + # puts "mnt_full_path: #{mnt_full_path}" + # puts "mnt_full_path.to_s: #{mnt_full_path.to_s}" + + # if File.exists?(mnt_full_path.to_s) + # puts "The File #{mnt_full_path.to_s} exists" + # end + + # puts "going to trigger" + # Issue Trigger rebalance xattr begin XAttr.set(mnt_full_path.to_s, REBALANCE_XATTR, "1", no_follow: true) - rescue ex : IO::Error - # DHT raises EEXIST if rebalance is not required for a file - # If file is deleted in after directory listing and before calling this setxattr - if ex.os_error != Errno::EEXIST && ex.os_error != Errno::ENOENT - STDERR.puts "Failed to trigger rebalance. file=#{rel_path} Error=#{ex}" - end + # puts "setting xattr for: #{mnt_full_path.to_s}" + # rescue ex : IO::Error + # # DHT raises EEXIST if rebalance is not required for a file + # # If file is deleted in after directory listing and before calling this setxattr + # if ex.os_error != Errno::EEXIST && ex.os_error != Errno::ENOENT + # puts "error" + # STDERR.puts "Failed to trigger rebalance. file=#{rel_path} Error=#{ex}" + # end + rescue ex + puts "other err: #{ex.message}" end + # begin + # puts XAttr.get(mnt_full_path.to_s, REBALANCE_XATTR, no_follow: true) + # rescue ex + # puts ex.message + # end + + + # begin + # XAttr.set(mnt_full_path.to_s, "name", "v") + # rescue ex + # puts ex.message + # end + + # begin + # puts XAttr.get(mnt_full_path.to_s, "name") + # rescue ex + # puts ex.message + # end + # Increment if rebalance complete or rebalance not required # or if any other error. add_scanned_bytes(file_size, file_info.@stat.st_nlink) diff --git a/mgr/src/server/services/fix_layout.cr b/mgr/src/server/services/fix_layout.cr index e7663be8..35982751 100644 --- a/mgr/src/server/services/fix_layout.cr +++ b/mgr/src/server/services/fix_layout.cr @@ -9,7 +9,7 @@ class FixLayoutService < Service def initialize(pool_name, volume_name, storage_unit) @create_pid_file = true - @path = Path[PROGRAM_NAME].expand.to_s + @path = PROGRAM_NAME == "kadalu" ? PROGRAM_NAME : Path[PROGRAM_NAME].expand.to_s @id = "rebalance-fix-layout-#{storage_unit.path}" @pid_file = "/run/kadalu/#{@id}.pid" @args = [ diff --git a/mgr/src/server/services/migrate_data.cr b/mgr/src/server/services/migrate_data.cr index 3b6d14c9..4d30df3e 100644 --- a/mgr/src/server/services/migrate_data.cr +++ b/mgr/src/server/services/migrate_data.cr @@ -9,7 +9,7 @@ class MigrateDataService < Service def initialize(pool_name, volume_name, storage_unit) @create_pid_file = true - @path = Path[PROGRAM_NAME].expand.to_s + @path = PROGRAM_NAME == "kadalu" ? PROGRAM_NAME : Path[PROGRAM_NAME].expand.to_s @id = "rebalance-migrate-data-#{storage_unit.path}" @pid_file = "/run/kadalu/#{@id}.pid" @args = [ diff --git a/tests/all/volumes.t b/tests/all/volumes.t index 780cbaf9..4133090d 100644 --- a/tests/all/volumes.t +++ b/tests/all/volumes.t @@ -46,7 +46,6 @@ nodes.each do |node| TEST "mkdir -p /exports/vol19" TEST "mkdir -p /exports/vol20a" TEST "mkdir -p /exports/vol20b" - end USE_NODE nodes[0] @@ -241,7 +240,13 @@ TEST "kadalu volume delete DEV/vol19 --mode=script" # Tests for volume expansion # Distribute TEST "kadalu volume create DEV/vol15 server1:/exports/vol15/s1 server2:/exports/vol15/s2 server3:/exports/vol15/s3" +TEST "mkdir -p /mnt/vol15" +puts TEST "kadalu mount DEV/vol15 /mnt/vol15" +puts TEST "df /mnt/vol15" +TEST "mkdir /mnt/vol15/d1 /mnt/vol15/d2 /mnt/vol15/d3" +TEST "touch /mnt/vol15/d1/f{0..9}" TEST "kadalu volume expand DEV/vol15 server1:/exports/vol15/s1_e server2:/exports/vol15/s2_e server3:/exports/vol15/s3_e" + TEST "kadalu volume stop DEV/vol15 --mode=script" TEST "kadalu volume delete DEV/vol15 --mode=script" From 26e40b8e74f9bab2f083165e3c216b0e442f9bb8 Mon Sep 17 00:00:00 2001 From: Shree Vatsa N Date: Mon, 16 Jan 2023 10:43:03 +0530 Subject: [PATCH 3/4] Add tests Signed-off-by: Shree Vatsa N --- .github/workflows/on-pr-submit.yml | 6 +- mgr/src/cmds/rebalance_process.cr | 48 +----- mgr/src/server/plugins/volume_expand.cr | 9 -- mgr/src/server/plugins/volume_utils.cr | 4 +- tests/all/volumes.t | 203 +++++++++++++++++++++++- 5 files changed, 212 insertions(+), 58 deletions(-) diff --git a/.github/workflows/on-pr-submit.yml b/.github/workflows/on-pr-submit.yml index 8902f286..3ed198d3 100644 --- a/.github/workflows/on-pr-submit.yml +++ b/.github/workflows/on-pr-submit.yml @@ -37,8 +37,8 @@ jobs: run: | cd tests && ./build-container.sh - name: Setup Test environment - run: binnacle -v tests/setup.t + run: binnacle -vv tests/setup.t - name: Build and Install Storage manager to nodes/containers - run: VERSION="${{ github.ref_name }}" binnacle -v tests/install.t + run: VERSION="${{ github.ref_name }}" binnacle -vv tests/install.t - name: Run all Tests - run: binnacle -v tests/all + run: binnacle -vv tests/all diff --git a/mgr/src/cmds/rebalance_process.cr b/mgr/src/cmds/rebalance_process.cr index bc0a9f1f..b4343e7d 100644 --- a/mgr/src/cmds/rebalance_process.cr +++ b/mgr/src/cmds/rebalance_process.cr @@ -139,7 +139,6 @@ class Rebalancer next if @ignore_paths.includes?(rel_path.to_s) backend_full_path = Path.new(@backend_dir, rel_path) - # puts "backend_full_path: #{backend_full_path}" if File.directory?(backend_full_path) # TODO: Fix ENOENT issue in mount and remove list entries for, # Correct use of rebalance multiprocesses & improved time complexity. @@ -160,52 +159,17 @@ class Rebalancer next end - - # puts "@mount_dir: #{@mount_dir}" - # puts "rel_path: #{rel_path}" - # puts "mnt_full_path: #{mnt_full_path}" - # puts "mnt_full_path.to_s: #{mnt_full_path.to_s}" - - # if File.exists?(mnt_full_path.to_s) - # puts "The File #{mnt_full_path.to_s} exists" - # end - - # puts "going to trigger" - # Issue Trigger rebalance xattr begin XAttr.set(mnt_full_path.to_s, REBALANCE_XATTR, "1", no_follow: true) - # puts "setting xattr for: #{mnt_full_path.to_s}" - # rescue ex : IO::Error - # # DHT raises EEXIST if rebalance is not required for a file - # # If file is deleted in after directory listing and before calling this setxattr - # if ex.os_error != Errno::EEXIST && ex.os_error != Errno::ENOENT - # puts "error" - # STDERR.puts "Failed to trigger rebalance. file=#{rel_path} Error=#{ex}" - # end - rescue ex - puts "other err: #{ex.message}" + rescue ex : IO::Error + # DHT raises EEXIST if rebalance is not required for a file + # If file is deleted in after directory listing and before calling this setxattr + if ex.os_error != Errno::EEXIST && ex.os_error != Errno::ENOENT + STDERR.puts "Failed to trigger rebalance. file=#{rel_path} Error=#{ex}" + end end - # begin - # puts XAttr.get(mnt_full_path.to_s, REBALANCE_XATTR, no_follow: true) - # rescue ex - # puts ex.message - # end - - - # begin - # XAttr.set(mnt_full_path.to_s, "name", "v") - # rescue ex - # puts ex.message - # end - - # begin - # puts XAttr.get(mnt_full_path.to_s, "name") - # rescue ex - # puts ex.message - # end - # Increment if rebalance complete or rebalance not required # or if any other error. add_scanned_bytes(file_size, file_info.@stat.st_nlink) diff --git a/mgr/src/server/plugins/volume_expand.cr b/mgr/src/server/plugins/volume_expand.cr index 7e5e24c4..f2f044c7 100644 --- a/mgr/src/server/plugins/volume_expand.cr +++ b/mgr/src/server/plugins/volume_expand.cr @@ -6,15 +6,6 @@ require "../datastore/*" require "./ping" require "./volume_utils.cr" -ACTION_MANAGE_SERVICES = "manage_services" - -node_action ACTION_MANAGE_SERVICES do |data, _env| - services, volfiles, _ = VolumeRequestToNode.from_json(data) - save_volfiles(volfiles) - sighup_processes(services) - restart_shd_service_and_start_fix_layout_service(services) -end - put "/api/v1/pools/:pool_name/volumes" do |env| pool_name = env.params.url["pool_name"] diff --git a/mgr/src/server/plugins/volume_utils.cr b/mgr/src/server/plugins/volume_utils.cr index 9ab9b91f..565c0409 100644 --- a/mgr/src/server/plugins/volume_utils.cr +++ b/mgr/src/server/plugins/volume_utils.cr @@ -36,7 +36,7 @@ node_action ACTION_VOLUME_CREATE_STOPPED do |data, _env| end node_action ACTION_MANAGE_SERVICES do |data, _env| - services, volfiles, rollback_volume, action = VolumeRequestToNodeWithAction.from_json(data) + services, volfiles, _, action = VolumeRequestToNodeWithAction.from_json(data) save_volfiles(volfiles) sighup_processes(services) restart_shd_service_and_manage_rebalance_services(services, action) @@ -152,7 +152,6 @@ def restart_shd_service_and_manage_rebalance_services(services, action = "start" else svc.stop end - end end end @@ -579,4 +578,3 @@ def handle_volume_rebalance_start_stop(data, action) NodeResponse.new(true, "") end - diff --git a/tests/all/volumes.t b/tests/all/volumes.t index 4133090d..35cfff1c 100644 --- a/tests/all/volumes.t +++ b/tests/all/volumes.t @@ -244,27 +244,228 @@ TEST "mkdir -p /mnt/vol15" puts TEST "kadalu mount DEV/vol15 /mnt/vol15" puts TEST "df /mnt/vol15" TEST "mkdir /mnt/vol15/d1 /mnt/vol15/d2 /mnt/vol15/d3" -TEST "touch /mnt/vol15/d1/f{0..9}" +TEST "touch /mnt/vol15/d1/f{1..9}" +TEST "touch /mnt/vol15/d2/f{1..9}" +TEST "touch /mnt/vol15/d3/f{1..9}" TEST "kadalu volume expand DEV/vol15 server1:/exports/vol15/s1_e server2:/exports/vol15/s2_e server3:/exports/vol15/s3_e" +EQUAL "3", (TEST "ls /exports/vol15/s1_e/* -d | wc -l").strip, "Check for fix-layout in server1 s1 unit" +USE_NODE nodes[1] +EQUAL "3", (TEST "ls /exports/vol15/s2_e/* -d | wc -l").strip, "Check for fix-layout in server2 s2 unit" +USE_NODE nodes[2] +EQUAL "3", (TEST "ls /exports/vol15/s3_e/* -d | wc -l").strip, "Check for fix-layout in server3 s3 unit" + +USE_NODE nodes[0] +TEST "kadalu volume rebalance-start DEV/vol15" +TEST "sleep 3" + +EQUAL "3", (TEST "ls /exports/vol15/s1/d1/f3 /exports/vol15/s1/d2/f3 /exports/vol15/s1/d3/f3 | wc -l").strip, "Check for migrate-data in server1 s1 unit vol15" +EQUAL "5", (TEST "ls /exports/vol15/s1_e/d1/f4 /exports/vol15/s1_e/d2/f1 /exports/vol15/s1_e/d2/f5 /exports/vol15/s1_e/d3/f1 /exports/vol15/s1_e/d3/f5 | wc -l").strip, "Check for migrate-data in server1 s1_e unit" + +USE_NODE nodes[1] +EQUAL "3", (TEST "ls /exports/vol15/s2/d1/f2 /exports/vol15/s2/d2/f2 /exports/vol15/s2/d3/f2 | wc -l").strip, "Check for migrate-data in server2 s2 unit vol15" +EQUAL "9", (TEST "ls /exports/vol15/s2_e/d1/f6 /exports/vol15/s2_e/d1/f7 /exports/vol15/s2_e/d1/f9 /exports/vol15/s2_e/d2/f6 /exports/vol15/s2_e/d2/f7 /exports/vol15/s2_e/d2/f9 /exports/vol15/s2_e/d3/f6 /exports/vol15/s2_e/d3/f7 /exports/vol15/s2_e/d3/f9 | wc -l +").strip, "Check for migrate-data in server2 s2_e unit" + +USE_NODE nodes[2] +EQUAL "3", (TEST "ls /exports/vol15/s3/d1/f8 /exports/vol15/s3/d2/f4 /exports/vol15/s3/d3/f4 | wc -l ").strip, "Check for migrate-data in server3 s3 unit vol15" +EQUAL "4", (TEST "ls /exports/vol15/s3_e/d1/f1 /exports/vol15/s3_e/d1/f5 /exports/vol15/s3_e/d2/f8 /exports/vol15/s3_e/d3/f8 | wc -l").strip, "Check for migrate-data in server3 s3_e unit" + +USE_NODE nodes[0] +TEST "umount /mnt/vol15" +TEST "rmdir /mnt/vol15" TEST "kadalu volume stop DEV/vol15 --mode=script" TEST "kadalu volume delete DEV/vol15 --mode=script" # Replicate TEST "kadalu volume create DEV/vol16 replica server1:/exports/vol16/s1 server2:/exports/vol16/s2 server3:/exports/vol16/s3" +TEST "mkdir -p /mnt/vol16" +puts TEST "kadalu mount DEV/vol16 /mnt/vol16" +puts TEST "df /mnt/vol16" +TEST "mkdir /mnt/vol16/d1 /mnt/vol16/d2 /mnt/vol16/d3" +TEST "touch /mnt/vol16/d1/f{1..9}" +TEST "touch /mnt/vol16/d2/f{1..9}" +TEST "touch /mnt/vol16/d3/f{1..9}" + TEST "kadalu volume expand DEV/vol16 replica server1:/exports/vol16/s1_e server2:/exports/vol16/s2_e server3:/exports/vol16/s3_e" + +EQUAL "3", (TEST "ls /exports/vol16/s1_e/* -d | wc -l").strip, "Check for fix-layout in server1 s1 unit vol16" +USE_NODE nodes[1] +EQUAL "3", (TEST "ls /exports/vol16/s2_e/* -d | wc -l").strip, "Check for fix-layout in server2 s2 unit vol16" +USE_NODE nodes[2] +EQUAL "3", (TEST "ls /exports/vol16/s3_e/* -d | wc -l").strip, "Check for fix-layout in server3 s3 unit vol16" + +USE_NODE nodes[0] +TEST "kadalu volume rebalance-start DEV/vol16" +TEST "sleep 3" + +EQUAL "5", (TEST "ls /exports/vol16/s1/d1/ | wc -l").strip, "Check for migrate-data in server1 s1/d1 unit vol16" +EQUAL "4", (TEST "ls /exports/vol16/s1/d2/ | wc -l").strip, "Check for migrate-data in server1 s1/d2 unit vol16" +EQUAL "4", (TEST "ls /exports/vol16/s1/d3/ | wc -l").strip, "Check for migrate-data in server1 s1/d3 unit vol16" + +EQUAL "4", (TEST "ls /exports/vol16/s1_e/d1/ | wc -l").strip, "Check for migrate-data in server1 s1_e/d1 unit vol16" +EQUAL "5", (TEST "ls /exports/vol16/s1_e/d2/ | wc -l").strip, "Check for migrate-data in server1 s1_e/d2 unit vol16" +EQUAL "5", (TEST "ls /exports/vol16/s1_e/d3/ | wc -l").strip, "Check for migrate-data in server1 s1_e/d3 unit vol16" + +USE_NODE nodes[1] +EQUAL "5", (TEST "ls /exports/vol16/s2/d1/ | wc -l").strip, "Check for migrate-data in server2 s2/d1 unit vol16" +EQUAL "4", (TEST "ls /exports/vol16/s2/d2/ | wc -l").strip, "Check for migrate-data in server2 s2/d2 unit vol16" +EQUAL "4", (TEST "ls /exports/vol16/s2/d3/ | wc -l").strip, "Check for migrate-data in server2 s2/d3 unit vol16" + +EQUAL "4", (TEST "ls /exports/vol16/s2_e/d1/ | wc -l").strip, "Check for migrate-data in server2 s2_e/d1 unit vol16" +EQUAL "5", (TEST "ls /exports/vol16/s2_e/d2/ | wc -l").strip, "Check for migrate-data in server2 s2_e/d2 unit vol16" +EQUAL "5", (TEST "ls /exports/vol16/s2_e/d3/ | wc -l").strip, "Check for migrate-data in server2 s2_e/d3 unit vol16" + +USE_NODE nodes[2] +EQUAL "5", (TEST "ls /exports/vol16/s3/d1/ | wc -l").strip, "Check for migrate-data in server3 s3/d1 unit vol16" +EQUAL "4", (TEST "ls /exports/vol16/s3/d2/ | wc -l").strip, "Check for migrate-data in server3 s3/d2 unit vol16" +EQUAL "4", (TEST "ls /exports/vol16/s3/d3/ | wc -l").strip, "Check for migrate-data in server3 s3/d3 unit vol16" + +EQUAL "4", (TEST "ls /exports/vol16/s3_e/d1/ | wc -l").strip, "Check for migrate-data in server3 s3_e/d1 unit vol16" +EQUAL "5", (TEST "ls /exports/vol16/s3_e/d2/ | wc -l").strip, "Check for migrate-data in server3 s3_e/d2 unit vol16" +EQUAL "5", (TEST "ls /exports/vol16/s3_e/d3/ | wc -l").strip, "Check for migrate-data in server3 s3_e/d3 unit vol16" + +USE_NODE nodes[0] +TEST "umount /mnt/vol16" +TEST "rmdir /mnt/vol16" + TEST "kadalu volume stop DEV/vol16 --mode=script" TEST "kadalu volume delete DEV/vol16 --mode=script" # Distributed Replicate TEST "kadalu volume create DEV/vol17 replica server1:/exports/vol17/s1 server2:/exports/vol17/s2 server3:/exports/vol17/s3 replica server1:/exports/vol17/s4 server2:/exports/vol17/s5 server3:/exports/vol17/s6" +TEST "mkdir -p /mnt/vol17" +puts TEST "kadalu mount DEV/vol17 /mnt/vol17" +puts TEST "df /mnt/vol17" +TEST "mkdir /mnt/vol17/d1 /mnt/vol17/d2 /mnt/vol17/d3" +TEST "touch /mnt/vol17/d1/f{1..9}" +TEST "touch /mnt/vol17/d2/f{1..9}" +TEST "touch /mnt/vol17/d3/f{1..9}" + TEST "kadalu volume expand DEV/vol17 replica server1:/exports/vol17/s1_e server2:/exports/vol17/s2_e server3:/exports/vol17/s3_e replica server1:/exports/vol17/s4_e server2:/exports/vol17/s5_e server3:/exports/vol17/s6_e" + +EQUAL "3", (TEST "ls /exports/vol17/s1_e/* -d | wc -l").strip, "Check for fix-layout in server1 s1 unit vol17" +EQUAL "3", (TEST "ls /exports/vol17/s4_e/* -d | wc -l").strip, "Check for fix-layout in server1 s4 unit vol17" +USE_NODE nodes[1] +EQUAL "3", (TEST "ls /exports/vol17/s2_e/* -d | wc -l").strip, "Check for fix-layout in server2 s2 unit vol17" +EQUAL "3", (TEST "ls /exports/vol17/s5_e/* -d | wc -l").strip, "Check for fix-layout in server2 s5 unit vol17" +USE_NODE nodes[2] +EQUAL "3", (TEST "ls /exports/vol17/s3_e/* -d | wc -l").strip, "Check for fix-layout in server3 s3 unit vol17" +EQUAL "3", (TEST "ls /exports/vol17/s6_e/* -d | wc -l").strip, "Check for fix-layout in server3 s6 unit vol17" + +USE_NODE nodes[0] +TEST "kadalu volume rebalance-start DEV/vol17" +TEST "sleep 3" + +EQUAL "2", (TEST "ls /exports/vol17/s1/d1/ | wc -l").strip, "Check for migrate-data in server1 s1/d1 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s1/d2/ | wc -l").strip, "Check for migrate-data in server1 s1/d2 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s1/d3/ | wc -l").strip, "Check for migrate-data in server1 s1/d3 unit vol17" + +EQUAL "2", (TEST "ls /exports/vol17/s4/d1/ | wc -l").strip, "Check for migrate-data in server1 s4/d1 unit vol17" +EQUAL "3", (TEST "ls /exports/vol17/s4/d2/ | wc -l").strip, "Check for migrate-data in server1 s4/d2 unit vol17" +EQUAL "3", (TEST "ls /exports/vol17/s4/d3/ | wc -l").strip, "Check for migrate-data in server1 s4/d3 unit vol17" + +EQUAL "2", (TEST "ls /exports/vol17/s1_e/d1/ | wc -l").strip, "Check for migrate-data in server1 s1_e/d1 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s1_e/d2/ | wc -l").strip, "Check for migrate-data in server1 s1_e/d2 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s1_e/d3/ | wc -l").strip, "Check for migrate-data in server1 s1_e/d3 unit vol17" + +EQUAL "3", (TEST "ls /exports/vol17/s4_e/d1/ | wc -l").strip, "Check for migrate-data in server1 s4_e/d1 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s4_e/d2/ | wc -l").strip, "Check for migrate-data in server1 s4_e/d2 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s4_e/d3/ | wc -l").strip, "Check for migrate-data in server1 s4_e/d3 unit vol17" + +USE_NODE nodes[1] +EQUAL "2", (TEST "ls /exports/vol17/s2/d1/ | wc -l").strip, "Check for migrate-data in server2 s2/d1 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s2/d2/ | wc -l").strip, "Check for migrate-data in server2 s2/d2 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s2/d3/ | wc -l").strip, "Check for migrate-data in server2 s2/d3 unit vol17" + +EQUAL "2", (TEST "ls /exports/vol17/s5/d1/ | wc -l").strip, "Check for migrate-data in server2 s5/d1 unit vol17" +EQUAL "3", (TEST "ls /exports/vol17/s5/d2/ | wc -l").strip, "Check for migrate-data in server2 s5/d2 unit vol17" +EQUAL "3", (TEST "ls /exports/vol17/s5/d3/ | wc -l").strip, "Check for migrate-data in server2 s5/d3 unit vol17" + +EQUAL "2", (TEST "ls /exports/vol17/s2_e/d1/ | wc -l").strip, "Check for migrate-data in server2 s2_e/d1 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s2_e/d2/ | wc -l").strip, "Check for migrate-data in server2 s2_e/d2 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s2_e/d3/ | wc -l").strip, "Check for migrate-data in server2 s2_e/d3 unit vol17" + +EQUAL "3", (TEST "ls /exports/vol17/s5_e/d1/ | wc -l").strip, "Check for migrate-data in server2 s5_e/d1 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s5_e/d2/ | wc -l").strip, "Check for migrate-data in server2 s5_e/d2 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s5_e/d3/ | wc -l").strip, "Check for migrate-data in server2 s5_e/d3 unit vol17" + +USE_NODE nodes[2] +EQUAL "2", (TEST "ls /exports/vol17/s3/d1/ | wc -l").strip, "Check for migrate-data in server3 s3/d1 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s3/d2/ | wc -l").strip, "Check for migrate-data in server3 s3/d2 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s3/d3/ | wc -l").strip, "Check for migrate-data in server3 s3/d3 unit vol17" + +EQUAL "2", (TEST "ls /exports/vol17/s6/d1/ | wc -l").strip, "Check for migrate-data in server3 s6/d1 unit vol17" +EQUAL "3", (TEST "ls /exports/vol17/s6/d2/ | wc -l").strip, "Check for migrate-data in server3 s6/d2 unit vol17" +EQUAL "3", (TEST "ls /exports/vol17/s6/d3/ | wc -l").strip, "Check for migrate-data in server3 s6/d3 unit vol17" + +EQUAL "2", (TEST "ls /exports/vol17/s3_e/d1/ | wc -l").strip, "Check for migrate-data in server3 s3_e/d1 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s3_e/d2/ | wc -l").strip, "Check for migrate-data in server3 s3_e/d2 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s3_e/d3/ | wc -l").strip, "Check for migrate-data in server3 s3_e/d3 unit vol17" + +EQUAL "3", (TEST "ls /exports/vol17/s6_e/d1/ | wc -l").strip, "Check for migrate-data in server3 s6_e/d1 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s6_e/d2/ | wc -l").strip, "Check for migrate-data in server3 s6_e/d2 unit vol17" +EQUAL "2", (TEST "ls /exports/vol17/s6_e/d3/ | wc -l").strip, "Check for migrate-data in server3 s6_e/d3 unit vol17" + +USE_NODE nodes[0] +TEST "umount /mnt/vol17" +TEST "rmdir /mnt/vol17" + TEST "kadalu volume stop DEV/vol17 --mode=script" TEST "kadalu volume delete DEV/vol17 --mode=script" # Disperse TEST "kadalu volume create DEV/vol18 data server1:/exports/vol18/s1 server2:/exports/vol18/s2 redundancy server3:/exports/vol18/s3" +TEST "mkdir -p /mnt/vol18" +puts TEST "kadalu mount DEV/vol18 /mnt/vol18" +puts TEST "df /mnt/vol18" +TEST "mkdir /mnt/vol18/d1 /mnt/vol18/d2 /mnt/vol18/d3" +TEST "touch /mnt/vol18/d1/f{1..9}" +TEST "touch /mnt/vol18/d2/f{1..9}" +TEST "touch /mnt/vol18/d3/f{1..9}" + TEST "kadalu volume expand DEV/vol18 data server1:/exports/vol18/s1_e server2:/exports/vol18/s2_e redundancy server3:/exports/vol18/s3_e" + +EQUAL "3", (TEST "ls /exports/vol18/s1_e/* -d | wc -l").strip, "Check for fix-layout in server1 s1 unit vol18" +USE_NODE nodes[1] +EQUAL "3", (TEST "ls /exports/vol18/s2_e/* -d | wc -l").strip, "Check for fix-layout in server2 s2 unit vol18" +USE_NODE nodes[2] +EQUAL "3", (TEST "ls /exports/vol18/s3_e/* -d | wc -l").strip, "Check for fix-layout in server3 s3 unit vol18" + +USE_NODE nodes[0] +TEST "kadalu volume rebalance-start DEV/vol18" +TEST "sleep 3" + +EQUAL "5", (TEST "ls /exports/vol18/s1/d1/ | wc -l").strip, "Check for migrate-data in server1 s1/d1 unit vol18" +EQUAL "4", (TEST "ls /exports/vol18/s1/d2/ | wc -l").strip, "Check for migrate-data in server1 s1/d2 unit vol18" +EQUAL "4", (TEST "ls /exports/vol18/s1/d3/ | wc -l").strip, "Check for migrate-data in server1 s1/d3 unit vol18" + +EQUAL "4", (TEST "ls /exports/vol18/s1_e/d1/ | wc -l").strip, "Check for migrate-data in server1 s1_e/d1 unit vol18" +EQUAL "5", (TEST "ls /exports/vol18/s1_e/d2/ | wc -l").strip, "Check for migrate-data in server1 s1_e/d2 unit vol18" +EQUAL "5", (TEST "ls /exports/vol18/s1_e/d3/ | wc -l").strip, "Check for migrate-data in server1 s1_e/d3 unit vol18" + +USE_NODE nodes[1] +EQUAL "5", (TEST "ls /exports/vol18/s2/d1/ | wc -l").strip, "Check for migrate-data in server2 s2/d1 unit vol18" +EQUAL "4", (TEST "ls /exports/vol18/s2/d2/ | wc -l").strip, "Check for migrate-data in server2 s2/d2 unit vol18" +EQUAL "4", (TEST "ls /exports/vol18/s2/d3/ | wc -l").strip, "Check for migrate-data in server2 s2/d3 unit vol18" + +EQUAL "4", (TEST "ls /exports/vol18/s2_e/d1/ | wc -l").strip, "Check for migrate-data in server2 s2_e/d1 unit vol18" +EQUAL "5", (TEST "ls /exports/vol18/s2_e/d2/ | wc -l").strip, "Check for migrate-data in server2 s2_e/d2 unit vol18" +EQUAL "5", (TEST "ls /exports/vol18/s2_e/d3/ | wc -l").strip, "Check for migrate-data in server2 s2_e/d3 unit vol18" + +USE_NODE nodes[2] +EQUAL "5", (TEST "ls /exports/vol18/s3/d1/ | wc -l").strip, "Check for migrate-data in server3 s3/d1 unit vol18" +EQUAL "4", (TEST "ls /exports/vol18/s3/d2/ | wc -l").strip, "Check for migrate-data in server3 s3/d2 unit vol18" +EQUAL "4", (TEST "ls /exports/vol18/s3/d3/ | wc -l").strip, "Check for migrate-data in server3 s3/d3 unit vol18" + +EQUAL "4", (TEST "ls /exports/vol18/s3_e/d1/ | wc -l").strip, "Check for migrate-data in server3 s3_e/d1 unit vol18" +EQUAL "5", (TEST "ls /exports/vol18/s3_e/d2/ | wc -l").strip, "Check for migrate-data in server3 s3_e/d2 unit vol18" +EQUAL "5", (TEST "ls /exports/vol18/s3_e/d3/ | wc -l").strip, "Check for migrate-data in server3 s3_e/d3 unit vol18" + +USE_NODE nodes[0] +TEST "umount /mnt/vol18" +TEST "rmdir /mnt/vol18" + TEST "kadalu volume stop DEV/vol18 --mode=script" TEST "kadalu volume delete DEV/vol18 --mode=script" From f1ac911c57edeb4a442c2345f904f6649bdb2e6f Mon Sep 17 00:00:00 2001 From: Shree Vatsa N Date: Tue, 17 Jan 2023 14:00:51 +0530 Subject: [PATCH 4/4] Address review comments 1 Signed-off-by: Shree Vatsa N --- mgr/src/cmds/rebalance_start_stop.cr | 47 +++++++++++++++++++++++++ mgr/src/cmds/volumes.cr | 46 ------------------------ mgr/src/server/plugins/volume_utils.cr | 31 +--------------- mgr/src/server/services/fix_layout.cr | 2 +- mgr/src/server/services/migrate_data.cr | 2 +- tests/all/volumes.t | 8 ++--- 6 files changed, 54 insertions(+), 82 deletions(-) create mode 100644 mgr/src/cmds/rebalance_start_stop.cr diff --git a/mgr/src/cmds/rebalance_start_stop.cr b/mgr/src/cmds/rebalance_start_stop.cr new file mode 100644 index 00000000..894f3262 --- /dev/null +++ b/mgr/src/cmds/rebalance_start_stop.cr @@ -0,0 +1,47 @@ +require "./helpers" + +command "rebalance.start", "Start Rebalancing Kadalu Storage volume" do |parser, _| + parser.banner = "Usage: kadalu rebalance start POOL/VOLNAME [arguments]" +end + +handler "rebalance.start" do |args| + begin + command_error "Pool/Volname is required" if args.pos_args.size == 0 + args.pool_name, volume_name = pool_and_volume_name(args.pos_args.size > 0 ? args.pos_args[0] : "") + api_call(args, "Failed to start rebalancing of volume") do |client| + volume = client.pool(args.pool_name).volume(volume_name).rebalance_start + + handle_json_output(volume, args) + + puts "Rebalance of Volume #{volume.name} started" + end + rescue ex : InvalidVolumeRequest + STDERR.puts "Starting of volume rebalance failed" + STDERR.puts ex + exit 1 + end +end + +command "rebalance.stop", "Stop Rebalancing Kadalu Storage volume" do |parser, _| + parser.banner = "Usage: kadalu rebalance stop POOL/VOLNAME [arguments]" +end + +handler "rebalance.stop" do |args| + begin + command_error "Pool/Volname is required" if args.pos_args.size == 0 + args.pool_name, volume_name = pool_and_volume_name(args.pos_args.size > 0 ? args.pos_args[0] : "") + next unless (args.script_mode || yes("Are you sure you want to stop rebalancing of volume? [y/N]")) + + api_call(args, "Failed to stop rebalancing of volume.") do |client| + volume = client.pool(args.pool_name).volume(volume_name).rebalance_stop + + handle_json_output(volume, args) + + puts "Rebalancing of Volume #{volume.name} stopped" + end + rescue ex : InvalidVolumeRequest + STDERR.puts "Stopping of volume rebalance failed" + STDERR.puts ex + exit 1 + end +end diff --git a/mgr/src/cmds/volumes.cr b/mgr/src/cmds/volumes.cr index 24709d2b..c9addbf1 100644 --- a/mgr/src/cmds/volumes.cr +++ b/mgr/src/cmds/volumes.cr @@ -347,49 +347,3 @@ handler "volume.expand" do |args| exit 1 end end - -command "volume.rebalance-start", "Start Rebalancing Kadalu Storage volume" do |parser, _| - parser.banner = "Usage: kadalu volume rebalance-start POOL/VOLNAME [arguments]" -end - -handler "volume.rebalance-start" do |args| - begin - command_error "Pool/Volname is required" if args.pos_args.size == 0 - args.pool_name, volume_name = pool_and_volume_name(args.pos_args.size > 0 ? args.pos_args[0] : "") - api_call(args, "Failed to start rebalancing of volume") do |client| - volume = client.pool(args.pool_name).volume(volume_name).rebalance_start - - handle_json_output(volume, args) - - puts "Rebalance of Volume #{volume.name} started" - end - rescue ex : InvalidVolumeRequest - STDERR.puts "Starting of volume rebalance failed" - STDERR.puts ex - exit 1 - end -end - -command "volume.rebalance-stop", "Stop Rebalancing Kadalu Storage volume" do |parser, _| - parser.banner = "Usage: kadalu volume rebalance-stop POOL/VOLNAME [arguments]" -end - -handler "volume.rebalance-stop" do |args| - begin - command_error "Pool/Volname is required" if args.pos_args.size == 0 - args.pool_name, volume_name = pool_and_volume_name(args.pos_args.size > 0 ? args.pos_args[0] : "") - next unless (args.script_mode || yes("Are you sure you want to stop rebalancing of volume? [y/N]")) - - api_call(args, "Failed to stop rebalancing of volume.") do |client| - volume = client.pool(args.pool_name).volume(volume_name).rebalance_stop - - handle_json_output(volume, args) - - puts "Rebalancing of Volume #{volume.name} stopped" - end - rescue ex : InvalidVolumeRequest - STDERR.puts "Stopping of volume rebalance failed" - STDERR.puts ex - exit 1 - end -end diff --git a/mgr/src/server/plugins/volume_utils.cr b/mgr/src/server/plugins/volume_utils.cr index 565c0409..fd3f0580 100644 --- a/mgr/src/server/plugins/volume_utils.cr +++ b/mgr/src/server/plugins/volume_utils.cr @@ -145,7 +145,7 @@ def restart_shd_service_and_manage_rebalance_services(services, action = "start" if svc.name == "shdservice" svc.restart elsif svc.name == "fixlayoutservice" || svc.name == "migratedataservice" - status_file_path = "/var/lib/kadalu/#{svc.id.gsub("/", "%2F")}.json" + status_file_path = "/var/lib/kadalu/#{svc.id}.json" FileUtils.rm(status_file_path) if File.exists?(status_file_path) if action == "start" svc.start @@ -549,32 +549,3 @@ def add_migrate_data_service(services, pool_name, volume_name, node, storage_uni services end - -def handle_volume_rebalance_start_stop(data, action) - services, volfiles, _ = VolumeRequestToNode.from_json(data) - - if action == "start" && !volfiles[GlobalConfig.local_node.id]?.nil? - Dir.mkdir_p(Path.new(GlobalConfig.workdir, "volfiles")) - volfiles[GlobalConfig.local_node.id].each do |volfile| - File.write(Path.new(GlobalConfig.workdir, "volfiles", "#{volfile.name}.vol"), volfile.content) - end - end - - unless services[GlobalConfig.local_node.id]?.nil? - # TODO: Hard coded path change? - Dir.mkdir_p("/var/log/kadalu") - Dir.mkdir_p("/run/kadalu") - services[GlobalConfig.local_node.id].each do |service| - svc = Service.from_json(service.to_json) - if svc.name == "migratedataservice" - if action == "start" - svc.start - else - svc.stop - end - end - end - end - - NodeResponse.new(true, "") -end diff --git a/mgr/src/server/services/fix_layout.cr b/mgr/src/server/services/fix_layout.cr index 35982751..9275182a 100644 --- a/mgr/src/server/services/fix_layout.cr +++ b/mgr/src/server/services/fix_layout.cr @@ -10,7 +10,7 @@ class FixLayoutService < Service @create_pid_file = true @path = PROGRAM_NAME == "kadalu" ? PROGRAM_NAME : Path[PROGRAM_NAME].expand.to_s - @id = "rebalance-fix-layout-#{storage_unit.path}" + @id = "rebalance-fix-layout-#{storage_unit.path.gsub("/", "%2F")}" @pid_file = "/run/kadalu/#{@id}.pid" @args = [ "_rebalance", "--fix-layout", diff --git a/mgr/src/server/services/migrate_data.cr b/mgr/src/server/services/migrate_data.cr index 4d30df3e..9a972ec3 100644 --- a/mgr/src/server/services/migrate_data.cr +++ b/mgr/src/server/services/migrate_data.cr @@ -10,7 +10,7 @@ class MigrateDataService < Service @create_pid_file = true @path = PROGRAM_NAME == "kadalu" ? PROGRAM_NAME : Path[PROGRAM_NAME].expand.to_s - @id = "rebalance-migrate-data-#{storage_unit.path}" + @id = "rebalance-migrate-data-#{storage_unit.path.gsub("/", "%2F")}" @pid_file = "/run/kadalu/#{@id}.pid" @args = [ "_rebalance", "--migrate-data", diff --git a/tests/all/volumes.t b/tests/all/volumes.t index 35cfff1c..5a935006 100644 --- a/tests/all/volumes.t +++ b/tests/all/volumes.t @@ -256,7 +256,7 @@ USE_NODE nodes[2] EQUAL "3", (TEST "ls /exports/vol15/s3_e/* -d | wc -l").strip, "Check for fix-layout in server3 s3 unit" USE_NODE nodes[0] -TEST "kadalu volume rebalance-start DEV/vol15" +TEST "kadalu rebalance start DEV/vol15" TEST "sleep 3" EQUAL "3", (TEST "ls /exports/vol15/s1/d1/f3 /exports/vol15/s1/d2/f3 /exports/vol15/s1/d3/f3 | wc -l").strip, "Check for migrate-data in server1 s1 unit vol15" @@ -296,7 +296,7 @@ USE_NODE nodes[2] EQUAL "3", (TEST "ls /exports/vol16/s3_e/* -d | wc -l").strip, "Check for fix-layout in server3 s3 unit vol16" USE_NODE nodes[0] -TEST "kadalu volume rebalance-start DEV/vol16" +TEST "kadalu rebalance start DEV/vol16" TEST "sleep 3" EQUAL "5", (TEST "ls /exports/vol16/s1/d1/ | wc -l").strip, "Check for migrate-data in server1 s1/d1 unit vol16" @@ -354,7 +354,7 @@ EQUAL "3", (TEST "ls /exports/vol17/s3_e/* -d | wc -l").strip, "Check for fix-la EQUAL "3", (TEST "ls /exports/vol17/s6_e/* -d | wc -l").strip, "Check for fix-layout in server3 s6 unit vol17" USE_NODE nodes[0] -TEST "kadalu volume rebalance-start DEV/vol17" +TEST "kadalu rebalance start DEV/vol17" TEST "sleep 3" EQUAL "2", (TEST "ls /exports/vol17/s1/d1/ | wc -l").strip, "Check for migrate-data in server1 s1/d1 unit vol17" @@ -433,7 +433,7 @@ USE_NODE nodes[2] EQUAL "3", (TEST "ls /exports/vol18/s3_e/* -d | wc -l").strip, "Check for fix-layout in server3 s3 unit vol18" USE_NODE nodes[0] -TEST "kadalu volume rebalance-start DEV/vol18" +TEST "kadalu rebalance start DEV/vol18" TEST "sleep 3" EQUAL "5", (TEST "ls /exports/vol18/s1/d1/ | wc -l").strip, "Check for migrate-data in server1 s1/d1 unit vol18"