compatiable for ray lastest 2.9.3 version (#60)

* compatiable for ray lastest 2.9.3 version

* build in temp ray branch with internal symbol in

* upgrade testng version to skip security

* use bazel 5.4.1 for building

* deb bazel package name fix

* fix unused set variables

* use aliyun mirror source list

* use bazel tools new jdk deps

* streaming java api reference upgrade 1.x to 2.9.3

* rename default resource from MEM to memory

* use internal exported taykey symbol

* no more for unseal ray pack

* python config parse fix

* update function descriptor and use master commit linking to ray

* fix stats report unittest in streaming

* fix cpp event service test namespace abm

.github/workflows/debian_manylinux_build.yaml

@@ -23,11 +23,11 @@ jobs:
 
     - name: Apt get update and Install bazel
       run:  |
-            apt-get update
-            apt-get install -yq wget openjdk-8-jdk zlib1g-dev zip git
             cp /etc/apt/sources.list /etc/apt/sources.list.bak
             cp .github/sources.list /etc/apt/sources.list
             apt-get update
+            apt-get install -yq wget openjdk-8-jdk zlib1g-dev zip git
+            apt-get update
             apt-get install -yq gcc g++
             sh scripts/install-bazel.sh
 

.gitignore

@@ -187,6 +187,7 @@ streaming/python/raystreaming/_streaming.so
 streaming/python/raystreaming/generated
 streaming/python/raystreaming/jars
 streaming/python/build
+streaming/python/dist
 
 # gitpod cache related
 .pip-cache/

scripts/format.sh

@@ -6,9 +6,9 @@ cd $CURRENT_DIR || exit
 ls .
 ROOT="$(git rev-parse --show-toplevel)"
 
-CLANG_FORMAT_VERSION_REQUIRED="12.0.0"
+CLANG_FORMAT_VERSION_REQUIRED="12.0.1"
 CLANG_FORMAT_VERSION=""
-BLACK_VERSION_REQUIRED="21.12b0"
+BLACK_VERSION_REQUIRED="22.3.0"
 BLACK_VERSION=""
 MAVEN_VERSION=""
 SHELLCHECK_VERSION_REQUIRED="0.7.1"

scripts/install-bazel.sh

@@ -24,11 +24,11 @@ esac
 echo "platform is ${platform}"
 
 if [ "${platform}" = "darwin" ]; then
-    wget "https://github.com/bazelbuild/bazel/releases/download/5.1.0/bazel-5.1.0-installer-darwin-x86_64.sh" -O bazel-5.1.0-installer-darwin-x86_64.sh
-    sh bazel-5.1.0-installer-darwin-x86_64.sh
+    wget "https://github.com/bazelbuild/bazel/releases/download/5.4.1/bazel-5.4.1-installer-darwin-x86_64.sh" -O bazel-5.4.1-installer-darwin-x86_64.sh
+    sh bazel-5.4.1-installer-darwin-x86_64.sh
 else
-    wget "https://github.com/bazelbuild/bazel/releases/download/5.1.0/bazel_5.1.0-linux-x86_64.deb" -O bazel_5.1.0-linux-x86_64.deb
-    dpkg -i bazel_5.1.0-linux-x86_64.deb
+    wget "https://github.com/bazelbuild/bazel/releases/download/5.4.1/bazel_5.4.1-linux-x86_64.deb" -O bazel_5.4.1-linux-x86_64.deb
+    dpkg -i bazel_5.4.1-linux-x86_64.deb
 fi
 
 bazel --version

scripts/prepare-format-tools.sh

@@ -14,7 +14,7 @@ install_clang_format() {
 }
 
 install_black_format() {
-    pip3 install black==21.12b0
+    pip3 install black==22.3.0
 }
 
 install_shell_check() {

streaming/.bazelrc

@@ -4,29 +4,39 @@ build --enable_platform_specific_config
 # On       Windows, provide: BAZEL_SH, and BAZEL_LLVM (if using clang-cl)
 # On all platforms, provide: PYTHON3_BIN_PATH=python
 ###############################################################################
-build --action_env=PATH
+build:windows --action_env=PATH
 # For --compilation_mode=dbg, consider enabling checks in the standard library as well (below).
 build --compilation_mode=opt
 # Using C++ 17 on all platforms.
+build:linux --host_cxxopt="-std=c++17"
+build:macos --host_cxxopt="-std=c++17"
+build:clang-cl --host_cxxopt="-std=c++17"
+build:msvc-cl --host_cxxopt="/std:c++17"
+build:windows --host_cxxopt="/std:c++17"
 build:linux --cxxopt="-std=c++17"
 build:macos --cxxopt="-std=c++17"
 build:clang-cl --cxxopt="-std=c++17"
 build:msvc-cl --cxxopt="/std:c++17"
 build:windows --cxxopt="/std:c++17"
 # This workaround is needed to prevent Bazel from compiling the same file twice (once PIC and once not).
-build:linux --force_pic
-build:macos --force_pic
+build:linux --copt="-fPIC"
+build:macos --copt="-fPIC"
 build:clang-cl --compiler=clang-cl
 build:msvc-cl --compiler=msvc-cl
 # `LC_ALL` and `LANG` is needed for cpp worker tests, because they will call "ray start".
 # If we don't add them, python's `click` library will raise an error.
-build --action_env=LC_ALL
-build --action_env=LANG
+test --test_env=LC_ALL
+test --test_env=LANG
 # Allow C++ worker tests to execute "ray start" with the correct version of Python.
-build --action_env=VIRTUAL_ENV
-build --action_env=PYENV_VIRTUAL_ENV
-build --action_env=PYENV_VERSION
-build --action_env=PYENV_SHELL
+test --test_env=VIRTUAL_ENV
+test --test_env=PYENV_VIRTUAL_ENV
+test --test_env=PYENV_VERSION
+test --test_env=PYENV_SHELL
+# Do not send usage stats to the server for tests
+test --test_env=RAY_USAGE_STATS_REPORT_URL="http://127.0.0.1:8000"
+# Enable cluster mode for OSX and Windows. By default, Ray
+# will not allow multinode OSX and Windows clusters.
+test --test_env=RAY_ENABLE_WINDOWS_OR_OSX_CLUSTER="1"
 # This is needed for some core tests to run correctly
 build:windows --enable_runfiles
 # TODO(mehrdadn): Revert the "-\\.(asm|S)$" exclusion when this Bazel bug
@@ -39,9 +49,12 @@ build:clang-cl --per_file_copt="-\\.(asm|S)$@-Werror"
 build:msvc-cl     --per_file_copt="-\\.(asm|S)$@-WX"
 # Ignore warnings for protobuf generated files and external projects.
 build --per_file_copt="\\.pb\\.cc$@-w"
-build --per_file_copt="-\\.(asm|S)$,external/.*@-w"
-#build --per_file_copt="external/.*@-Wno-unused-result"
-# Ignore minor warnings for host tools, which we generally can't control
+build:linux --per_file_copt="-\\.(asm|S)$,external/.*@-w,-Wno-error=implicit-function-declaration"
+build:macos --per_file_copt="-\\.(asm|S)$,external/.*@-w,-Wno-error=implicit-function-declaration"
+# Ignore warnings for host tools, which we generally can't control.
+# Ideally we only want to ignore warnings for external project
+# but the current bazel version doesn't support host_per_file_copt yet.
+build:macos --host_copt="-w"
 build:clang-cl --host_copt="-Wno-inconsistent-missing-override"
 build:clang-cl --host_copt="-Wno-microsoft-unqualified-friend"
 # This workaround is needed due to https://github.com/bazelbuild/bazel/issues/4341
@@ -78,7 +91,7 @@ build:manylinux2010 --linkopt="-lrt"
 #build:windows --cxxopt="-D_ITERATOR_DEBUG_LEVEL=2"
 
 # LLVM (clang & libc++) build flags common across Linux installations and systems.
-# On Ubuntu, the remaining configurations can be generated by running ci/travis/install-llvm-binaries.sh
+# On Ubuntu, the remaining configurations can be generated by running ci/env/install-llvm-binaries.sh
 build:llvm --action_env=CXXFLAGS=-stdlib=libc++
 build:llvm --action_env=LDFLAGS=-stdlib=libc++
 build:llvm --action_env=BAZEL_CXXOPTS=-stdlib=libc++
@@ -87,14 +100,16 @@ build:llvm --action_env=BAZEL_LINKOPTS=-lm:-pthread
 build:llvm --define force_libcpp=enabled
 
 # Thread sanitizer configuration:
+build:tsan --per_file_copt="-bazel-ray/external/com_github_antirez_redis/.*$@-fsanitize=thread"
+build:tsan --per_file_copt="-bazel-ray/external/com_github_antirez_redis/.*$@-DTHREAD_SANITIZER"
 build:tsan --strip=never
-build:tsan --copt -fsanitize=thread
-build:tsan --copt -DTHREAD_SANITIZER
 build:tsan --copt -O2
 build:tsan --copt -g
 build:tsan --copt -fno-omit-frame-pointer
 build:tsan --copt -Wno-uninitialized
 build:tsan --linkopt -fsanitize=thread
+build:tsan --cxxopt="-D_RAY_TSAN_BUILD"
+build:tsan --no//:jemalloc_flag
 # This config is only for running TSAN with LLVM toolchain on Linux.
 build:tsan-clang --config=tsan
 build:tsan-clang --config=llvm
@@ -107,6 +122,7 @@ build:asan --copt -fsanitize=address
 build:asan --copt -DADDRESS_SANITIZER
 build:asan --copt -fno-omit-frame-pointer
 build:asan --linkopt -fsanitize=address
+build:asan --no//:jemalloc_flag
 test:asan --jobs=1
 test:asan --test_env=ASAN_OPTIONS="detect_leaks=0"
 # This config is only for running ASAN with LLVM toolchain on Linux.
@@ -142,10 +158,8 @@ build:ci --show_task_finish
 build:ci --ui_actions_shown=1024
 build:ci --show_timestamps
 build:ci-travis --disk_cache=~/ray-bazel-cache
-build:ci-travis --remote_cache="https://storage.googleapis.com/ray-bazel-cache"
 build:ci-github --experimental_repository_cache_hardlinks # GitHub Actions has low disk space, so prefer hardlinks there.
 build:ci-github --disk_cache=~/ray-bazel-cache
-build:ci-github --remote_cache="https://storage.googleapis.com/ray-bazel-cache"
 test:ci --flaky_test_attempts=3
 # Disable test result caching because py_test under Bazel can import from outside of sandbox, but Bazel only looks at
 # declared dependencies to determine if a result should be cached. More details at:
@@ -176,17 +190,25 @@ test:segfault --run_under="bash -c 'unset GREP_OPTIONS && if ! grep -q -o Micros
 # Debug build:
 build:debug -c dbg
 build:debug --copt="-g"
+build:debug --copt -fno-omit-frame-pointer
 build:debug --strip="never"
 
 # Undefined Behavior Sanitizer
 build:ubsan --strip=never
-build:ubsan --copt -fsanitize=undefined
 build:ubsan --copt -fno-sanitize=vptr
 build:ubsan --copt -fno-sanitize-recover=all
 build:ubsan --copt -g
 build:ubsan --linkopt -fsanitize=undefined
 build:ubsan --linkopt -fno-sanitize-recover=all
+build:ubsan --per_file_copt="-external/com_github_grpc_grpc/.*@-fsanitize=undefined"
 
 # Import local specific llvm config options, which can be generated by
-# ci/travis/install-llvm-dependencies.sh
+# ci/env/install-llvm-dependencies.sh
 try-import %workspace%/.llvm-local.bazelrc
+
+# Even with sandbox mode bazel prioritizes system headers over the ones in the sandbox.
+# It picks up the system headers when someone has protobuf installed via Homebrew.
+# Work around for https://github.com/bazelbuild/bazel/issues/8053
+build:macos --sandbox_block_path=/usr/local/
+# This option controls whether javac checks for missing direct dependencies.
+build --experimental_strict_java_deps=off

streaming/BUILD.bazel

@@ -2,7 +2,7 @@
 # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html
 
 load("@rules_proto_grpc//python:defs.bzl", "python_proto_compile")
-load("@com_github_ray_project_ray//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS", "copy_to_workspace")
+load("@com_github_ray_project_ray//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS", "copy_to_workspace", "ray_cc_library")
 load("@rules_python//python:defs.bzl", "py_library")
 load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library")
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
@@ -68,7 +68,7 @@ cc_binary(
     copts = COPTS,
     linkshared = 1,
     visibility = ["//visibility:public"],
-    deps = ["@com_github_ray_project_ray//:ray_util"],
+    deps = ["@com_github_ray_project_ray//src/ray/util"],
 )
 
 cc_binary(
@@ -305,7 +305,7 @@ test_common_deps = [
     "@com_github_ray_project_ray//:exported_internal",
     ":streaming_lib",
     "@com_github_ray_project_ray//:ray_common",
-    "@com_github_ray_project_ray//:ray_util",
+    "@com_github_ray_project_ray//src/ray/util",
     "@com_github_ray_project_ray//:core_worker_lib",
 ]
 

streaming/WORKSPACE

@@ -2,12 +2,12 @@ workspace(name = "com_github_ray_streaming")
 
 # LOAD RAY WORKSPACE
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
-ray_version = "cba26cc83f6b5b8a2ff166594a65cb74c0ec8740"
+ray_version = "5e7b3c3de1efacb937701c4eebea64b6a02996be"
 http_archive(
         name="com_github_ray_project_ray",
         strip_prefix = "ray-{}".format(ray_version),
         urls = ["https://github.com/ray-project/ray/archive/{}.zip".format(ray_version)],
-        sha256 = "8b5bd40cf28372f9684d49ea5627f9ae247854882309f0f261378ba5d849ad24",
+        sha256 = "9059707ca9bf63e7521586acb0e7ddbd9e10ac1d0ecfc95274f6b6b08c67eb1c",
 )
 
 http_archive(
@@ -27,6 +27,15 @@ ray_deps_setup()
 load("@com_github_ray_project_ray//bazel:ray_deps_build_all.bzl", "ray_deps_build_all")
 
 ray_deps_build_all()
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python3_9",
+    python_version = "3.9",
+    register_toolchains = False,
+)
+
 load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
 
 grpc_extra_deps()
@@ -39,4 +48,4 @@ load("@bazel_skylib//lib:versions.bzl", "versions")
 
 # When the bazel version is updated, make sure to update it
 # in setup.py as well.
-versions.check(minimum_bazel_version = "4.2.1")
+versions.check(minimum_bazel_version = "5.4.1")

streaming/buildtest.sh

@@ -4,6 +4,12 @@ TMP_LOG_OUTPUT="$script_dir"/tmp/logs
 
 mkdir -p "$TMP_LOG_OUTPUT"
 
+if [[ "$VIRTUAL_ENV" != "" ]]; then
+  INVENV=1
+else
+  INVENV=0
+fi
+
 function suppress_output()
 {
   "$script_dir"/../scripts/suppress_output "$@"
@@ -22,6 +28,12 @@ function create_py_env()
 
 function init()
 {
+    if [ $INVENV = 1 ]
+    then
+        echo "Already in vritual env, reuse $VIRTUAL_ENV"
+        return 0
+    fi
+
     pushd "$script_dir" || exit
     PY3_DIR=$script_dir/../py3
     if [ -d $PY3_DIR ]

streaming/java/dependencies.bzl

@@ -28,13 +28,13 @@ def gen_streaming_java_deps():
             maven.artifact(
                 group = "io.ray",
                 artifact = "ray-api",
-                version = "1.13.0",
+                version = "2.9.3",
                 neverlink = True
             ),
             maven.artifact(
                 group = "io.ray",
                 artifact = "ray-runtime",
-                version = "1.13.0",
+                version = "2.9.3",
                 neverlink = True
             ),
             maven.artifact(

streaming/java/pom.xml

@@ -57,7 +57,7 @@
   <properties>
     <java.version>1.8</java.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <ray.version>2.0.0</ray.version>
+    <ray.version>2.9.3</ray.version>
     <streaming.version>0.0.1</streaming.version>
     <mockito.version>1.10.19</mockito.version>
   </properties>
@@ -101,7 +101,7 @@
       <dependency>
         <groupId>org.testng</groupId>
         <artifactId>testng</artifactId>
-        <version>7.3.0</version>
+        <version>7.5.1</version>
         <exclusions>
           <exclusion>
             <groupId>org.yaml</groupId>

streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java

@@ -121,7 +121,7 @@ public Set<IndependentOperatorDescriptor> getIndependentOperators() {
   public void stop() {
     // for single-process mode, we need to close all the async thread, so we can't close the ray env
     // for single-process mode here
-    if (Ray.isInitialized() && !Ray.getRuntimeContext().isSingleProcess()) {
+    if (Ray.isInitialized() && !Ray.getRuntimeContext().isLocalMode()) {
       ClusterStarter.stopCluster();
     }
   }

streaming/java/streaming-api/src/test/java/io/ray/streaming/api/stream/StreamTest.java

@@ -151,15 +151,15 @@ public void testStreamContextWithIndependentOperators() {
     streamingContext
         .withIndependentOperator(className, moduleName, Language.PYTHON)
         .setParallelism(2)
-        .setResource(ImmutableMap.of(ResourceKey.MEM.name(), 2500D))
+        .setResource(ImmutableMap.of(ResourceKey.memory.name(), 2500D))
         .setConfig(ImmutableMap.of("k2", "v2"))
         .setLazyScheduling();
 
     // invalid(is duplicated)
     streamingContext
         .withIndependentOperator(className, moduleName, Language.PYTHON)
         .setParallelism(4)
-        .setResource(ImmutableMap.of(ResourceKey.MEM.name(), 2600D))
+        .setResource(ImmutableMap.of(ResourceKey.memory.name(), 2600D))
         .setConfig(ImmutableMap.of("k3", "v3"));
 
     Set<IndependentOperatorDescriptor> result = streamingContext.getIndependentOperators();
@@ -172,7 +172,7 @@ public void testStreamContextWithIndependentOperators() {
         Assert.assertEquals(independentOperatorDescriptor.getLanguage(), Language.PYTHON);
         Assert.assertEquals(independentOperatorDescriptor.getParallelism(), 2);
         Assert.assertEquals(
-            (double) independentOperatorDescriptor.getResource().get(ResourceKey.MEM.name()),
+            (double) independentOperatorDescriptor.getResource().get(ResourceKey.memory.name()),
             2500D);
         Assert.assertEquals(independentOperatorDescriptor.getConfig().get("k2"), "v2");
         Assert.assertTrue(independentOperatorDescriptor.isLazyScheduling());

streaming/java/streaming-common/src/main/java/io/ray/streaming/common/config/ResourceConfig.java

@@ -85,7 +85,7 @@ public interface ResourceConfig extends Config {
    *
    * @return buffer size
    */
-  @DefaultValue(value = "1600")
+  @DefaultValue(value = "500")
   @Key(value = WORKER_MEM)
   double workerMemMbRequired();
 

streaming/java/streaming-common/src/main/java/io/ray/streaming/common/config/converter/LogDirConverter.java

@@ -1,7 +1,7 @@
 package io.ray.streaming.common.config.converter;
 
 import io.ray.api.Ray;
-import io.ray.runtime.RayRuntimeInternal;
+import io.ray.runtime.AbstractRayRuntime;
 import java.lang.reflect.Method;
 import org.aeonbits.owner.Converter;
 
@@ -13,7 +13,7 @@ public class LogDirConverter implements Converter<String> {
   public String convert(Method method, String input) {
     if (input.isEmpty()) {
       return Ray.isInitialized()
-          ? ((RayRuntimeInternal) Ray.internal()).getRayConfig().logDir
+          ? ((AbstractRayRuntime) Ray.internal()).getRayConfig().logDir
           : DEFAULT_LOG_DIR;
     }
     return input;

streaming/java/streaming-common/src/main/java/io/ray/streaming/common/enums/ResourceKey.java

@@ -10,7 +10,7 @@ public enum ResourceKey {
   GPU("GPU"),
 
   /** Memory resource. */
-  MEM("MEM"),
+  memory("memory"),
 
   /** Disk storage size resource. */
   DISK("DISK"),