From 7e03dcb35369505da989a9886913e1a6a49867e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Szuma?= <56085570+Rados13@users.noreply.github.com> Date: Mon, 30 Oct 2023 13:04:01 +0100 Subject: [PATCH] [RTC 389] Add DNSpoll distribution strategy (#109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add DNS strategy * Add CI for DNS cluster * Changes after review * Fix credo issues * Add spec for read_boolean * Remove JF_DIST_NODE_BASENAME * Update lib/jellyfish/config_reader.ex Co-authored-by: Michał Śledź * Update lib/jellyfish/config_reader.ex Co-authored-by: Michał Śledź * Update test/jellyfish/config_reader_test.exs Co-authored-by: Michał Śledź * Update test/jellyfish/config_reader_test.exs Co-authored-by: Michał Śledź * Update lib/jellyfish/config_reader.ex Co-authored-by: Michał Śledź * Update lib/jellyfish/config_reader.ex Co-authored-by: Michał Śledź * Remove spec * Add do_read_nodes_list_config --------- Co-authored-by: Michał Śledź --- .circleci/config.yml | 4 +- docker-compose-dns.yaml | 87 ++++++++++++ ...r-compose.yaml => docker-compose-epmd.yaml | 0 lib/jellyfish/application.ex | 6 +- lib/jellyfish/config_reader.ex | 133 ++++++++++++++---- mix.exs | 7 +- test/jellyfish/config_reader_test.exs | 77 +++++++++- 7 files changed, 276 insertions(+), 38 deletions(-) create mode 100644 docker-compose-dns.yaml rename docker-compose.yaml => docker-compose-epmd.yaml (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21614a08..748b4c7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,7 +16,9 @@ jobs: executor: machine_executor_amd64 steps: - checkout - - run: docker compose run test + - run: docker compose -f docker-compose-epmd.yaml up test --exit-code-from test + - run: docker compose -f docker-compose-epmd.yaml down + - run: docker compose -f docker-compose-dns.yaml up test --exit-code-from test test: docker: diff --git a/docker-compose-dns.yaml b/docker-compose-dns.yaml new file mode 100644 index 00000000..241f9c50 --- /dev/null +++ b/docker-compose-dns.yaml @@ -0,0 +1,87 @@ +version: "3" + +x-jellyfish-template: &jellyfish-template + build: . + environment: &jellyfish-environment + JF_SERVER_API_TOKEN: "development" + JF_DIST_ENABLED: "true" + JF_DIST_STRATEGY_NAME: "DNS" + restart: on-failure + +services: + test: + image: membraneframeworklabs/docker_membrane + command: + - sh + - -c + - | + cd app/ + mix deps.get + MIX_ENV=ci mix test --only cluster + volumes: + - .:/app + - /app/_build + - /app/deps + networks: + dns-network: + aliases: + - dnsmasq + depends_on: + - app1 + - app2 + - dnsmasq + + app1: + <<: *jellyfish-template + environment: + <<: *jellyfish-environment + JF_HOST: "localhost:4001" + JF_PORT: 4001 + JF_DIST_NODE_NAME: app@172.28.1.2 + JF_DIST_QUERY: app.dns-network + ports: + - 4001:4001 + networks: + dns-network: + ipv4_address: 172.28.1.2 + aliases: + - app.dns-network + dns: + - 172.28.1.4:5353 + + app2: + container_name: name + <<: *jellyfish-template + environment: + <<: *jellyfish-environment + JF_HOST: "localhost:4002" + JF_PORT: 4002 + JF_DIST_NODE_NAME: app@172.28.1.3 + JF_DIST_QUERY: app.dns-network + ports: + - 4002:4002 + networks: + dns-network: + # Register IP under alias + ipv4_address: 172.28.1.3 + aliases: + - app.dns-network + dns: + - 172.28.1.4:5353 + + dnsmasq: + image: andyshinn/dnsmasq:2.78 + volumes: + - /etc/resolv.conf:/etc/resolv.dnsmasq.conf + command: -d -q --no-hosts --no-resolv --no-poll --server 127.0.0.11 --listen-address 0.0.0.0 --port 5353 --log-queries --log-facility=- --address=/./127.0.0.11 + networks: + dns-network: + ipv4_address: 172.28.1.4 + aliases: + - dnsmasq + +networks: + dns-network: + ipam: + config: + - subnet: 172.28.1.0/24 diff --git a/docker-compose.yaml b/docker-compose-epmd.yaml similarity index 100% rename from docker-compose.yaml rename to docker-compose-epmd.yaml diff --git a/lib/jellyfish/application.ex b/lib/jellyfish/application.ex index 570d0c8e..b3bf7fba 100644 --- a/lib/jellyfish/application.ex +++ b/lib/jellyfish/application.ex @@ -81,9 +81,9 @@ defmodule Jellyfish.Application do end topologies = [ - epmd_cluster: [ - strategy: Cluster.Strategy.Epmd, - config: [hosts: dist_config[:nodes]] + cluster: [ + strategy: dist_config[:strategy], + config: dist_config[:strategy_config] ] ] diff --git a/lib/jellyfish/config_reader.ex b/lib/jellyfish/config_reader.ex index 8be28006..467c8018 100644 --- a/lib/jellyfish/config_reader.ex +++ b/lib/jellyfish/config_reader.ex @@ -78,34 +78,6 @@ defmodule Jellyfish.ConfigReader do end end - def read_dist_config() do - if read_boolean("JF_DIST_ENABLED") do - node_name_value = System.get_env("JF_DIST_NODE_NAME") - cookie_value = System.get_env("JF_DIST_COOKIE", "jellyfish_cookie") - nodes_value = System.get_env("JF_DIST_NODES", "") - - unless node_name_value do - raise "JF_DIST_ENABLED has been set but JF_DIST_NODE_NAME remains unset." - end - - node_name = parse_node_name(node_name_value) - cookie = parse_cookie(cookie_value) - nodes = parse_nodes(nodes_value) - - if nodes == [] do - Logger.warning(""" - JF_DIST_ENABLED has been set but JF_DIST_NODES remains unset. - Make sure that at least one of your Jellyfish instances - has JF_DIST_NODES set. - """) - end - - [enabled: true, node_name: node_name, cookie: cookie, nodes: nodes] - else - [enabled: false, node_name: nil, cookie: nil, nodes: []] - end - end - def read_webrtc_config() do webrtc_used = read_boolean("JF_WEBRTC_USED") @@ -128,7 +100,98 @@ defmodule Jellyfish.ConfigReader do end end - defp parse_node_name(node_name), do: String.to_atom(node_name) + def read_dist_config() do + dist_enabled? = read_boolean("JF_DIST_ENABLED") + dist_strategy = System.get_env("JF_DIST_STRATEGY_NAME") + node_name_value = System.get_env("JF_DIST_NODE_NAME") + cookie_value = System.get_env("JF_DIST_COOKIE", "jellyfish_cookie") + + cond do + is_nil(dist_enabled?) or not dist_enabled? -> + [enabled: false, strategy: nil, node_name: nil, cookie: nil, strategy_config: nil] + + dist_strategy == "NODES_LIST" or is_nil(dist_strategy) -> + do_read_nodes_list_config(node_name_value, cookie_value) + + dist_strategy == "DNS" -> + do_read_dns_config(node_name_value, cookie_value) + + true -> + raise """ + JF_DIST_ENABLED has been set but unknown JF_DIST_STRATEGY was provided. + Availabile strategies are EPMD or DNS, provided strategy name was: "#{dist_strategy}" + """ + end + end + + defp do_read_nodes_list_config(node_name_value, cookie_value) do + nodes_value = System.get_env("JF_DIST_NODES", "") + + unless node_name_value do + raise "JF_DIST_ENABLED has been set but JF_DIST_NODE_NAME remains unset." + end + + node_name = parse_node_name(node_name_value) + cookie = parse_cookie(cookie_value) + nodes = parse_nodes(nodes_value) + + if nodes == [] do + Logger.warning(""" + NODES_LIST strategy requires JF_DIST_NODES to be set + by at least one Jellyfish instace. This instance has JF_DIST_NODES unset. + """) + end + + [ + enabled: true, + strategy: Cluster.Strategy.Epmd, + node_name: node_name, + cookie: cookie, + strategy_config: [hosts: nodes] + ] + end + + defp do_read_dns_config(node_name_value, cookie_value) do + unless node_name_value do + raise "JF_DIST_ENABLED has been set but JF_DIST_NODE_NAME remains unset." + end + + node_name = parse_node_name(node_name_value) + cookie = parse_cookie(cookie_value) + + query_value = System.get_env("JF_DIST_QUERY") + + unless query_value do + raise "JF_DIST_QUERY is required by DNS strategy" + end + + [node_basename, _ip_addres_or_fqdn | []] = String.split(node_name_value, "@") + + polling_interval = parse_polling_interval() + + [ + enabled: true, + strategy: Cluster.Strategy.DNSPoll, + node_name: node_name, + cookie: cookie, + strategy_config: [ + polling_interval: polling_interval, + query: query_value, + node_basename: node_basename + ] + ] + end + + defp parse_node_name(node_name) do + case String.split(node_name, "@") do + [_node_basename, _ip_addres_or_fqdn | []] -> + String.to_atom(node_name) + + _other -> + raise "JF_DIST_NODE_NAME has to be in form of @. Got: #{node_name}" + end + end + defp parse_cookie(cookie_value), do: String.to_atom(cookie_value) defp parse_nodes(nodes_value) do @@ -136,4 +199,16 @@ defmodule Jellyfish.ConfigReader do |> String.split(" ", trim: true) |> Enum.map(&String.to_atom(&1)) end + + defp parse_polling_interval() do + env_value = System.get_env("JF_DIST_POLLING_INTERVAL", "5000") + + case Integer.parse(env_value) do + {polling_interval, ""} when polling_interval > 0 -> + polling_interval + + _other -> + raise "`JF_DIST_POLLING_INTERVAL` must be a positivie integer. Got: #{env_value}" + end + end end diff --git a/mix.exs b/mix.exs index b082edd4..b4dd6fd4 100644 --- a/mix.exs +++ b/mix.exs @@ -97,7 +97,12 @@ defmodule Jellyfish.MixProject do "api.spec": &generate_api_spec/1, test: ["test --exclude cluster"], "test.cluster": ["test --only cluster"], - "test.cluster.ci": ["cmd docker compose run test; docker compose down"] + "test.cluster.ci": [ + "cmd docker compose -f docker-compose-epmd.yaml up test; docker compose -f docker-compose-epmd.yaml down" + ], + "test.cluster.dns.ci": [ + "cmd docker compose -f docker-compose-dns.yaml up test; docker compose -f docker-compose-dns.yaml down" + ] ] end diff --git a/test/jellyfish/config_reader_test.exs b/test/jellyfish/config_reader_test.exs index 9aceafb1..f1e75d72 100644 --- a/test/jellyfish/config_reader_test.exs +++ b/test/jellyfish/config_reader_test.exs @@ -112,36 +112,105 @@ defmodule Jellyfish.ConfigReaderTest do end end - test "read_dist_config/0" do + test "read_dist_config/0 NODES_LIST" do with_env ["JF_DIST_ENABLED", "JF_DIST_COOKIE", "JF_DIST_NODE_NAME", "JF_DIST_NODES"] do assert ConfigReader.read_dist_config() == [ enabled: false, + strategy: nil, node_name: nil, cookie: nil, - nodes: [] + strategy_config: nil ] System.put_env("JF_DIST_ENABLED", "true") assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end System.put_env("JF_DIST_COOKIE", "testcookie") assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end + System.put_env("JF_DIST_NODE_NAME", "testnodename@abc@def") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end + System.put_env("JF_DIST_NODE_NAME", "testnodename") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end System.put_env("JF_DIST_NODE_NAME", "testnodename@127.0.0.1") assert ConfigReader.read_dist_config() == [ enabled: true, + strategy: Cluster.Strategy.Epmd, node_name: :"testnodename@127.0.0.1", cookie: :testcookie, - nodes: [] + strategy_config: [hosts: []] ] System.put_env("JF_DIST_NODES", "testnodename1@127.0.0.1 testnodename2@127.0.0.1") assert ConfigReader.read_dist_config() == [ enabled: true, + strategy: Cluster.Strategy.Epmd, + node_name: :"testnodename@127.0.0.1", + cookie: :testcookie, + strategy_config: [hosts: [:"testnodename1@127.0.0.1", :"testnodename2@127.0.0.1"]] + ] + end + end + + test "read_dist_config/0 DNS" do + with_env [ + "JF_DIST_ENABLED", + "JF_DIST_COOKIE", + "JF_DIST_NODE_NAME", + "JF_DIST_NODES", + "JF_DIST_STRATEGY_NAME" + ] do + assert ConfigReader.read_dist_config() == [ + enabled: false, + strategy: nil, + node_name: nil, + cookie: nil, + strategy_config: nil + ] + + System.put_env("JF_DIST_ENABLED", "true") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end + System.put_env("JF_DIST_STRATEGY_NAME", "DNS") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end + System.put_env("JF_DIST_COOKIE", "testcookie") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end + System.put_env("JF_DIST_NODE_NAME", "testnodename@127.0.0.1") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end + System.put_env("JF_DIST_QUERY", "my-app.example.com") + + assert ConfigReader.read_dist_config() == [ + enabled: true, + strategy: Cluster.Strategy.DNSPoll, node_name: :"testnodename@127.0.0.1", cookie: :testcookie, - nodes: [:"testnodename1@127.0.0.1", :"testnodename2@127.0.0.1"] + strategy_config: [ + polling_interval: 5_000, + query: "my-app.example.com", + node_basename: "testnodename" + ] ] + + System.put_env( + "JF_DIST_POLLING_INTERVAL", + "10000" + ) + + assert ConfigReader.read_dist_config() == [ + enabled: true, + strategy: Cluster.Strategy.DNSPoll, + node_name: :"testnodename@127.0.0.1", + cookie: :testcookie, + strategy_config: [ + polling_interval: 10_000, + query: "my-app.example.com", + node_basename: "testnodename" + ] + ] + + System.put_env("JF_DIST_POLLING_INTERVAL", "abcd") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end + System.put_env("JF_DIST_POLLING_INTERVAL", "-25") + assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end end end end