[RTC 389] Add DNSpoll distribution strategy (#109)

* Add DNS strategy * Add CI for DNS cluster * Changes after review * Fix credo issues * Add spec for read_boolean * Remove JF_DIST_NODE_BASENAME * Update lib/jellyfish/config_reader.ex Co-authored-by: Michał Śledź <[email protected]> * Update lib/jellyfish/config_reader.ex Co-authored-by: Michał Śledź <[email protected]> * Update test/jellyfish/config_reader_test.exs Co-authored-by: Michał Śledź <[email protected]> * Update test/jellyfish/config_reader_test.exs Co-authored-by: Michał Śledź <[email protected]> * Update lib/jellyfish/config_reader.ex Co-authored-by: Michał Śledź <[email protected]> * Update lib/jellyfish/config_reader.ex Co-authored-by: Michał Śledź <[email protected]> * Remove spec * Add do_read_nodes_list_config --------- Co-authored-by: Michał Śledź <[email protected]>
fishjam-dev · Oct 30, 2023 · 7e03dcb · 7e03dcb
1 parent bf1809a
commit 7e03dcb
Show file tree

Hide file tree

Showing 7 changed files with 276 additions and 38 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -16,7 +16,9 @@ jobs:
     executor: machine_executor_amd64
     steps:
       - checkout
-      - run: docker compose run test
+      - run: docker compose -f docker-compose-epmd.yaml up test --exit-code-from test
+      - run: docker compose -f docker-compose-epmd.yaml down 
+      - run: docker compose -f docker-compose-dns.yaml up test --exit-code-from test
 
   test:
     docker:

diff --git a/docker-compose-dns.yaml b/docker-compose-dns.yaml
@@ -0,0 +1,87 @@
+version: "3"
+
+x-jellyfish-template: &jellyfish-template
+  build: .
+  environment: &jellyfish-environment
+    JF_SERVER_API_TOKEN: "development"
+    JF_DIST_ENABLED: "true"
+    JF_DIST_STRATEGY_NAME: "DNS"
+  restart: on-failure
+
+services:
+  test:
+    image: membraneframeworklabs/docker_membrane
+    command:
+      - sh
+      - -c
+      - |
+        cd app/ 
+        mix deps.get 
+        MIX_ENV=ci mix test --only cluster
+    volumes:
+      - .:/app
+      - /app/_build
+      - /app/deps
+    networks:
+      dns-network:
+        aliases:
+          - dnsmasq
+    depends_on:
+      - app1
+      - app2
+      - dnsmasq
+
+  app1:
+    <<: *jellyfish-template
+    environment:
+      <<: *jellyfish-environment
+      JF_HOST: "localhost:4001"
+      JF_PORT: 4001
+      JF_DIST_NODE_NAME: [email protected]
+      JF_DIST_QUERY: app.dns-network
+    ports:
+      - 4001:4001
+    networks:
+      dns-network:
+        ipv4_address: 172.28.1.2
+        aliases:
+          - app.dns-network
+    dns:
+      - 172.28.1.4:5353
+
+  app2:
+    container_name: name
+    <<: *jellyfish-template
+    environment:
+      <<: *jellyfish-environment
+      JF_HOST: "localhost:4002"
+      JF_PORT: 4002
+      JF_DIST_NODE_NAME: [email protected]
+      JF_DIST_QUERY: app.dns-network
+    ports:
+      - 4002:4002
+    networks:
+      dns-network:
+        # Register IP under alias
+        ipv4_address: 172.28.1.3
+        aliases:
+          - app.dns-network
+    dns:
+      - 172.28.1.4:5353
+
+  dnsmasq:
+    image: andyshinn/dnsmasq:2.78
+    volumes:
+      - /etc/resolv.conf:/etc/resolv.dnsmasq.conf
+    command: -d -q --no-hosts --no-resolv --no-poll --server 127.0.0.11 --listen-address 0.0.0.0 --port 5353 --log-queries --log-facility=- --address=/./127.0.0.11
+    networks:
+      dns-network:
+        ipv4_address: 172.28.1.4
+        aliases:
+          - dnsmasq
+
+networks:
+  dns-network:
+    ipam:
+      config:
+        - subnet: 172.28.1.0/24
diff --git a/docker-compose.yaml → docker-compose-epmd.yaml b/docker-compose.yaml → docker-compose-epmd.yaml
diff --git a/lib/jellyfish/application.ex b/lib/jellyfish/application.ex
@@ -81,9 +81,9 @@ defmodule Jellyfish.Application do
     end
 
     topologies = [
-      epmd_cluster: [
-        strategy: Cluster.Strategy.Epmd,
-        config: [hosts: dist_config[:nodes]]
+      cluster: [
+        strategy: dist_config[:strategy],
+        config: dist_config[:strategy_config]
       ]
     ]
 

diff --git a/lib/jellyfish/config_reader.ex b/lib/jellyfish/config_reader.ex
@@ -78,34 +78,6 @@ defmodule Jellyfish.ConfigReader do
     end
   end
 
-  def read_dist_config() do
-    if read_boolean("JF_DIST_ENABLED") do
-      node_name_value = System.get_env("JF_DIST_NODE_NAME")
-      cookie_value = System.get_env("JF_DIST_COOKIE", "jellyfish_cookie")
-      nodes_value = System.get_env("JF_DIST_NODES", "")
-
-      unless node_name_value do
-        raise "JF_DIST_ENABLED has been set but JF_DIST_NODE_NAME remains unset."
-      end
-
-      node_name = parse_node_name(node_name_value)
-      cookie = parse_cookie(cookie_value)
-      nodes = parse_nodes(nodes_value)
-
-      if nodes == [] do
-        Logger.warning("""
-        JF_DIST_ENABLED has been set but JF_DIST_NODES remains unset.
-        Make sure that at least one of your Jellyfish instances
-        has JF_DIST_NODES set.
-        """)
-      end
-
-      [enabled: true, node_name: node_name, cookie: cookie, nodes: nodes]
-    else
-      [enabled: false, node_name: nil, cookie: nil, nodes: []]
-    end
-  end
-
   def read_webrtc_config() do
     webrtc_used = read_boolean("JF_WEBRTC_USED")
 
@@ -128,12 +100,115 @@ defmodule Jellyfish.ConfigReader do
     end
   end
 
-  defp parse_node_name(node_name), do: String.to_atom(node_name)
+  def read_dist_config() do
+    dist_enabled? = read_boolean("JF_DIST_ENABLED")
+    dist_strategy = System.get_env("JF_DIST_STRATEGY_NAME")
+    node_name_value = System.get_env("JF_DIST_NODE_NAME")
+    cookie_value = System.get_env("JF_DIST_COOKIE", "jellyfish_cookie")
+
+    cond do
+      is_nil(dist_enabled?) or not dist_enabled? ->
+        [enabled: false, strategy: nil, node_name: nil, cookie: nil, strategy_config: nil]
+
+      dist_strategy == "NODES_LIST" or is_nil(dist_strategy) ->
+        do_read_nodes_list_config(node_name_value, cookie_value)
+
+      dist_strategy == "DNS" ->
+        do_read_dns_config(node_name_value, cookie_value)
+
+      true ->
+        raise """
+        JF_DIST_ENABLED has been set but unknown JF_DIST_STRATEGY was provided.
+        Availabile strategies are EPMD or DNS, provided strategy name was: "#{dist_strategy}"
+        """
+    end
+  end
+
+  defp do_read_nodes_list_config(node_name_value, cookie_value) do
+    nodes_value = System.get_env("JF_DIST_NODES", "")
+
+    unless node_name_value do
+      raise "JF_DIST_ENABLED has been set but JF_DIST_NODE_NAME remains unset."
+    end
+
+    node_name = parse_node_name(node_name_value)
+    cookie = parse_cookie(cookie_value)
+    nodes = parse_nodes(nodes_value)
+
+    if nodes == [] do
+      Logger.warning("""
+      NODES_LIST strategy requires JF_DIST_NODES to be set
+      by at least one Jellyfish instace. This instance has JF_DIST_NODES unset.
+      """)
+    end
+
+    [
+      enabled: true,
+      strategy: Cluster.Strategy.Epmd,
+      node_name: node_name,
+      cookie: cookie,
+      strategy_config: [hosts: nodes]
+    ]
+  end
+
+  defp do_read_dns_config(node_name_value, cookie_value) do
+    unless node_name_value do
+      raise "JF_DIST_ENABLED has been set but JF_DIST_NODE_NAME remains unset."
+    end
+
+    node_name = parse_node_name(node_name_value)
+    cookie = parse_cookie(cookie_value)
+
+    query_value = System.get_env("JF_DIST_QUERY")
+
+    unless query_value do
+      raise "JF_DIST_QUERY is required by DNS strategy"
+    end
+
+    [node_basename, _ip_addres_or_fqdn | []] = String.split(node_name_value, "@")
+
+    polling_interval = parse_polling_interval()
+
+    [
+      enabled: true,
+      strategy: Cluster.Strategy.DNSPoll,
+      node_name: node_name,
+      cookie: cookie,
+      strategy_config: [
+        polling_interval: polling_interval,
+        query: query_value,
+        node_basename: node_basename
+      ]
+    ]
+  end
+
+  defp parse_node_name(node_name) do
+    case String.split(node_name, "@") do
+      [_node_basename, _ip_addres_or_fqdn | []] ->
+        String.to_atom(node_name)
+
+      _other ->
+        raise "JF_DIST_NODE_NAME has to be in form of <nodename>@<hostname>. Got: #{node_name}"
+    end
+  end
+
   defp parse_cookie(cookie_value), do: String.to_atom(cookie_value)
 
   defp parse_nodes(nodes_value) do
     nodes_value
     |> String.split(" ", trim: true)
     |> Enum.map(&String.to_atom(&1))
   end
+
+  defp parse_polling_interval() do
+    env_value = System.get_env("JF_DIST_POLLING_INTERVAL", "5000")
+
+    case Integer.parse(env_value) do
+      {polling_interval, ""} when polling_interval > 0 ->
+        polling_interval
+
+      _other ->
+        raise "`JF_DIST_POLLING_INTERVAL` must be a positivie integer. Got: #{env_value}"
+    end
+  end
 end
diff --git a/mix.exs b/mix.exs
@@ -97,7 +97,12 @@ defmodule Jellyfish.MixProject do
       "api.spec": &generate_api_spec/1,
       test: ["test --exclude cluster"],
       "test.cluster": ["test --only cluster"],
-      "test.cluster.ci": ["cmd docker compose run test; docker compose down"]
+      "test.cluster.ci": [
+        "cmd docker compose -f docker-compose-epmd.yaml up test; docker compose -f docker-compose-epmd.yaml down"
+      ],
+      "test.cluster.dns.ci": [
+        "cmd docker compose -f docker-compose-dns.yaml up test; docker compose -f docker-compose-dns.yaml down"
+      ]
     ]
   end
 

diff --git a/test/jellyfish/config_reader_test.exs b/test/jellyfish/config_reader_test.exs
@@ -112,36 +112,105 @@ defmodule Jellyfish.ConfigReaderTest do
     end
   end
 
-  test "read_dist_config/0" do
+  test "read_dist_config/0 NODES_LIST" do
     with_env ["JF_DIST_ENABLED", "JF_DIST_COOKIE", "JF_DIST_NODE_NAME", "JF_DIST_NODES"] do
       assert ConfigReader.read_dist_config() == [
                enabled: false,
+               strategy: nil,
                node_name: nil,
                cookie: nil,
-               nodes: []
+               strategy_config: nil
              ]
 
       System.put_env("JF_DIST_ENABLED", "true")
       assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
       System.put_env("JF_DIST_COOKIE", "testcookie")
       assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
+      System.put_env("JF_DIST_NODE_NAME", "testnodename@abc@def")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
+      System.put_env("JF_DIST_NODE_NAME", "testnodename")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
       System.put_env("JF_DIST_NODE_NAME", "[email protected]")
 
       assert ConfigReader.read_dist_config() == [
                enabled: true,
+               strategy: Cluster.Strategy.Epmd,
                node_name: :"[email protected]",
                cookie: :testcookie,
-               nodes: []
+               strategy_config: [hosts: []]
              ]
 
       System.put_env("JF_DIST_NODES", "[email protected] [email protected]")
 
       assert ConfigReader.read_dist_config() == [
                enabled: true,
+               strategy: Cluster.Strategy.Epmd,
+               node_name: :"[email protected]",
+               cookie: :testcookie,
+               strategy_config: [hosts: [:"[email protected]", :"[email protected]"]]
+             ]
+    end
+  end
+
+  test "read_dist_config/0 DNS" do
+    with_env [
+      "JF_DIST_ENABLED",
+      "JF_DIST_COOKIE",
+      "JF_DIST_NODE_NAME",
+      "JF_DIST_NODES",
+      "JF_DIST_STRATEGY_NAME"
+    ] do
+      assert ConfigReader.read_dist_config() == [
+               enabled: false,
+               strategy: nil,
+               node_name: nil,
+               cookie: nil,
+               strategy_config: nil
+             ]
+
+      System.put_env("JF_DIST_ENABLED", "true")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
+      System.put_env("JF_DIST_STRATEGY_NAME", "DNS")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
+      System.put_env("JF_DIST_COOKIE", "testcookie")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
+      System.put_env("JF_DIST_NODE_NAME", "[email protected]")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
+      System.put_env("JF_DIST_QUERY", "my-app.example.com")
+
+      assert ConfigReader.read_dist_config() == [
+               enabled: true,
+               strategy: Cluster.Strategy.DNSPoll,
                node_name: :"[email protected]",
                cookie: :testcookie,
-               nodes: [:"[email protected]", :"[email protected]"]
+               strategy_config: [
+                 polling_interval: 5_000,
+                 query: "my-app.example.com",
+                 node_basename: "testnodename"
+               ]
              ]
+
+      System.put_env(
+        "JF_DIST_POLLING_INTERVAL",
+        "10000"
+      )
+
+      assert ConfigReader.read_dist_config() == [
+               enabled: true,
+               strategy: Cluster.Strategy.DNSPoll,
+               node_name: :"[email protected]",
+               cookie: :testcookie,
+               strategy_config: [
+                 polling_interval: 10_000,
+                 query: "my-app.example.com",
+                 node_basename: "testnodename"
+               ]
+             ]
+
+      System.put_env("JF_DIST_POLLING_INTERVAL", "abcd")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
+      System.put_env("JF_DIST_POLLING_INTERVAL", "-25")
+      assert_raise RuntimeError, fn -> ConfigReader.read_dist_config() end
     end
   end
 end