From 90723b96cf3877972af280a35acd15b78bffc6ba Mon Sep 17 00:00:00 2001 From: Adam Valenta Date: Sat, 6 Apr 2024 02:21:32 +0200 Subject: [PATCH 01/10] Upgrade to gradle 8.4 --- build.gradle | 2 +- gradle/copyJars.gradle | 10 +++++----- gradle/java.gradle | 2 +- gradle/javaIgnoreSymbolFile.gradle | 2 +- gradle/publish.gradle | 2 +- gradle/ubench.gradle | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- h2o-assemblies/genmodel/build.gradle | 2 +- h2o-assemblies/main/build.gradle | 2 +- h2o-assemblies/minimal/build.gradle | 2 +- h2o-assemblies/steam/build.gradle | 2 +- h2o-hadoop-2/assemblyjar.gradle | 4 ++-- h2o-hadoop-3/assemblyjar_common.gradle | 4 ++-- h2o-k8s-comp/build.gradle | 2 +- h2o-py-cloud-extensions/build.gradle | 1 + h2o-py-mlflow-flavor/build.gradle | 1 + 16 files changed, 22 insertions(+), 20 deletions(-) diff --git a/build.gradle b/build.gradle index 57107b67810f..be9bb55cfaf3 100644 --- a/build.gradle +++ b/build.gradle @@ -24,7 +24,7 @@ buildscript { //noinspection GroovyAssignabilityCheck dependencies { classpath 'org.ow2.asm:asm:5.1' - classpath 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0' + classpath 'com.github.johnrengelman:shadow:8.1.1' classpath 'org.gradle.api.plugins:gradle-nexus-plugin:0.7.1' classpath 'org.testng:testng:6.8' classpath 'me.champeau.gradle:jmh-gradle-plugin:0.5.2' diff --git a/gradle/copyJars.gradle b/gradle/copyJars.gradle index 47bff3798223..3409b0073ec0 100644 --- a/gradle/copyJars.gradle +++ b/gradle/copyJars.gradle @@ -3,14 +3,14 @@ tasks.withType(Jar) { Jar t -> t.doLast { copy { - from t.archivePath - into t.destinationDir + from t.archiveFile + into t.destinationDirectory // Here we expected standard JAR file archive name: // [baseName]-[appendix]-[version]-[classifier].[extension] - if (t.classifier!=null) { - rename("(.*)-${t.version}(.*)", '$1$2') + if (t.archiveClassifier!=null) { + rename("(.*)-${t.archiveVersion.get()}(.*)", '$1$2') } else { - rename("(.*)${t.version}-?(.*)", '$1$2') + rename("(.*)${t.archiveVersion.get()}-?(.*)", '$1$2') } } } diff --git a/gradle/java.gradle b/gradle/java.gradle index 4e35fff13277..e4b47163d0c3 100644 --- a/gradle/java.gradle +++ b/gradle/java.gradle @@ -11,7 +11,7 @@ compileJava { } task testJar(type: Jar, dependsOn: testClasses, group: "build") { - classifier = 'test' + archiveClassifier.set('test') from sourceSets.test.output } diff --git a/gradle/javaIgnoreSymbolFile.gradle b/gradle/javaIgnoreSymbolFile.gradle index 74c2eb45e35c..1528c27e0ef5 100644 --- a/gradle/javaIgnoreSymbolFile.gradle +++ b/gradle/javaIgnoreSymbolFile.gradle @@ -2,5 +2,5 @@ compileJava { // This only works when forking and passing 'javac' as an executable options.compilerArgs << '-XDignore.symbol.file' options.fork = true - options.forkOptions.executable = 'javac' + options.forkOptions.executable = "${System.properties['java.home']}/../bin/javac" } diff --git a/gradle/publish.gradle b/gradle/publish.gradle index 039ca026c8f5..bdcd66ddd07e 100644 --- a/gradle/publish.gradle +++ b/gradle/publish.gradle @@ -4,7 +4,7 @@ apply plugin: 'signing' // Default task to bundle sources code as jar task sourcesJar(type: Jar) { - classifier = 'sources' + archiveClassifier.set('sources') from sourceSets.main.allSource } diff --git a/gradle/ubench.gradle b/gradle/ubench.gradle index 2bfee1e1e848..32b5903cc843 100644 --- a/gradle/ubench.gradle +++ b/gradle/ubench.gradle @@ -76,7 +76,7 @@ task jmhPostProcess(dependsOn: "jmh") { } task jmhFatJar(type: ShadowJar) { - classifier = "jmh" + archiveClassifier.set('jmh') manifest { attributes "Main-Class": "org.openjdk.jmh.Main" } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ffed3a254e91..48c0a02ca419 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/h2o-assemblies/genmodel/build.gradle b/h2o-assemblies/genmodel/build.gradle index 346e1db6073a..c34aa72c8ca5 100644 --- a/h2o-assemblies/genmodel/build.gradle +++ b/h2o-assemblies/genmodel/build.gradle @@ -35,7 +35,7 @@ for (comp in optionalComponents) { shadowJar { mergeServiceFiles() - classifier = '' + archiveClassifier.set('') relocate 'com.google.gson', 'ai.h2o.com.google.gson' // Should be eventually removed: https://github.com/h2oai/h2o-3/issues/12372 relocate 'org.apache.commons.text', 'ai.h2o.org.apache.commons.text' relocate 'org.apache.commons.lang3', 'ai.h2o.org.apache.commons.lang3' diff --git a/h2o-assemblies/main/build.gradle b/h2o-assemblies/main/build.gradle index 6231736b5867..d836c896f4ad 100644 --- a/h2o-assemblies/main/build.gradle +++ b/h2o-assemblies/main/build.gradle @@ -92,7 +92,7 @@ dependencies { shadowJar { zip64 true mergeServiceFiles() - classifier = '' + archiveClassifier.set('') // CDH 5.3.0 provides joda-time v1.6 which is too old, shadow the library instead if (!project.hasProperty("jacocoCoverage")) { relocate 'org.joda.time', 'ai.h2o.org.joda.time' diff --git a/h2o-assemblies/minimal/build.gradle b/h2o-assemblies/minimal/build.gradle index 2a0fe0b54359..13e522ffe6c6 100644 --- a/h2o-assemblies/minimal/build.gradle +++ b/h2o-assemblies/minimal/build.gradle @@ -33,7 +33,7 @@ dependencies { shadowJar { mergeServiceFiles() - classifier = '' + archiveClassifier.set('') exclude 'META-INF/*.DSA' exclude 'META-INF/*.SF' exclude 'synchronize.properties' diff --git a/h2o-assemblies/steam/build.gradle b/h2o-assemblies/steam/build.gradle index 2f177ecd991b..a3609fdb3d13 100644 --- a/h2o-assemblies/steam/build.gradle +++ b/h2o-assemblies/steam/build.gradle @@ -89,7 +89,7 @@ dependencies { shadowJar { mergeServiceFiles() - classifier = '' + archiveClassifier.set('') exclude 'META-INF/*.DSA' exclude 'META-INF/*.SF' exclude 'synchronize.properties' diff --git a/h2o-hadoop-2/assemblyjar.gradle b/h2o-hadoop-2/assemblyjar.gradle index 6b69dce78b85..d87a359b6ff9 100644 --- a/h2o-hadoop-2/assemblyjar.gradle +++ b/h2o-hadoop-2/assemblyjar.gradle @@ -127,8 +127,8 @@ shadowJar { relocate 'org.apache.http', 'ai.h2o.org.apache.http' exclude hadoopShadowJarExcludes relocate 'com.google.common', 'ai.h2o.com.google.common' - baseName = 'h2odriver' - classifier = '' + archiveBaseName.set('h2odriver') + archiveClassifier.set('') manifest { attributes 'Main-Class': 'water.hadoop.h2odriver' } diff --git a/h2o-hadoop-3/assemblyjar_common.gradle b/h2o-hadoop-3/assemblyjar_common.gradle index ce5b3fc8c039..90f764655d98 100644 --- a/h2o-hadoop-3/assemblyjar_common.gradle +++ b/h2o-hadoop-3/assemblyjar_common.gradle @@ -90,8 +90,8 @@ shadowJar { exclude hadoopShadowJarExcludes relocate 'com.google.common', 'ai.h2o.com.google.common' relocate 'org.eclipse.jetty', 'ai.h2o.org.eclipse.jetty' - baseName = 'h2odriver' - classifier = '' + archiveBaseName.set('h2odriver') + archiveClassifier.set('') manifest { attributes 'Main-Class': 'water.hadoop.h2odriver' } diff --git a/h2o-k8s-comp/build.gradle b/h2o-k8s-comp/build.gradle index c607ff1b3636..6101bd0bdc52 100644 --- a/h2o-k8s-comp/build.gradle +++ b/h2o-k8s-comp/build.gradle @@ -19,7 +19,7 @@ dependencies { shadowJar { mergeServiceFiles() - classifier = '' + archiveClassifier.set('') exclude 'META-INF/*.DSA' exclude 'META-INF/*.SF' exclude 'synchronize.properties' diff --git a/h2o-py-cloud-extensions/build.gradle b/h2o-py-cloud-extensions/build.gradle index 2d4457445e08..b262686fe712 100644 --- a/h2o-py-cloud-extensions/build.gradle +++ b/h2o-py-cloud-extensions/build.gradle @@ -84,5 +84,6 @@ task cleanBuild(type: Delete) { // // Define the dependencies // +jar.dependsOn copySrcFiles clean.dependsOn cleanBuild, cleanUpTestResults build.dependsOn copyMainDist diff --git a/h2o-py-mlflow-flavor/build.gradle b/h2o-py-mlflow-flavor/build.gradle index 1edc3ccab43e..1b9608b908f6 100644 --- a/h2o-py-mlflow-flavor/build.gradle +++ b/h2o-py-mlflow-flavor/build.gradle @@ -59,5 +59,6 @@ task cleanBuild(type: Delete) { // // Define the dependencies // +jar.dependsOn copySrcFiles clean.dependsOn cleanBuild build.dependsOn copyMainDist From c8914a5daef63cac16861ff81c751fbb145d94f3 Mon Sep 17 00:00:00 2001 From: Adam Valenta Date: Wed, 10 Apr 2024 17:45:31 +0200 Subject: [PATCH 02/10] GH-15772 fix copyMainDist task for h2o-py --- h2o-py/build.gradle | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/h2o-py/build.gradle b/h2o-py/build.gradle index 094fce0ace76..03240602edfa 100644 --- a/h2o-py/build.gradle +++ b/h2o-py/build.gradle @@ -99,11 +99,17 @@ task buildDist(type: Exec, dependsOn: [verifyDependencies, createVersionFiles, c commandLine getOsSpecificCommandLine([pythonexe, "setup.py", "bdist_wheel"]) } -task copyMainDist(type: Copy, dependsOn: [buildDist]) { - from ("${buildDir}/main/") { - include "dist/**" +// I don't know why the task with type: Copy doesn't run after buildDist for gradle 8 even though its working for other projects (e.g. h2o-py-cloud-extension) +// I was able to properly copy the wheel only this way. +task copyMainDist(dependsOn: [buildDist]) { + doLast{ + copy { + from("${buildDir}/main/") { + include "dist/**" + } + into "${buildDir}" + } } - into "${buildDir}" } task buildClientDist(type: Exec, dependsOn: [verifyDependencies, createVersionFiles, copyClientSrcFiles]) { From ee2f8ee970229d750ef8ca2d5255d2bf8f99718e Mon Sep 17 00:00:00 2001 From: Adam Valenta Date: Thu, 11 Apr 2024 17:25:41 +0200 Subject: [PATCH 03/10] fix javac --- gradle/javaIgnoreSymbolFile.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/javaIgnoreSymbolFile.gradle b/gradle/javaIgnoreSymbolFile.gradle index 1528c27e0ef5..281b14562e05 100644 --- a/gradle/javaIgnoreSymbolFile.gradle +++ b/gradle/javaIgnoreSymbolFile.gradle @@ -2,5 +2,5 @@ compileJava { // This only works when forking and passing 'javac' as an executable options.compilerArgs << '-XDignore.symbol.file' options.fork = true - options.forkOptions.executable = "${System.properties['java.home']}/../bin/javac" + options.forkOptions.executable = "${System.env.JAVA_HOME}/bin/javac" } From 80791d6798cc0a825928d7ba5eb2133ef5bc93e0 Mon Sep 17 00:00:00 2001 From: Adam Valenta Date: Tue, 23 Apr 2024 17:09:00 +0200 Subject: [PATCH 04/10] fix javac for prismascan images --- gradle/javaIgnoreSymbolFile.gradle | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gradle/javaIgnoreSymbolFile.gradle b/gradle/javaIgnoreSymbolFile.gradle index 281b14562e05..c88b8fe1b21e 100644 --- a/gradle/javaIgnoreSymbolFile.gradle +++ b/gradle/javaIgnoreSymbolFile.gradle @@ -2,5 +2,10 @@ compileJava { // This only works when forking and passing 'javac' as an executable options.compilerArgs << '-XDignore.symbol.file' options.fork = true - options.forkOptions.executable = "${System.env.JAVA_HOME}/bin/javac" + if (System.env.JAVA_HOME != null) { + options.forkOptions.executable = "${System.env.JAVA_HOME}/bin/javac" + } else { + // JAVA_HOME is not set. Using expected javac location + options.forkOptions.executable = "/usr/bin/javac" + } } From 62009fed9a3f5fcf7a2013cece998ab67f2f5f05 Mon Sep 17 00:00:00 2001 From: krasinski <8573352+krasinski@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:46:41 +0200 Subject: [PATCH 05/10] remove force = true --- h2o-hadoop-2/assemblyjar.gradle | 28 +++++++++++++++++----------- h2o-hadoop-3/assemblyjar.gradle | 5 ++++- h2o-hadoop-3/assemblyjar_cdp.gradle | 5 ++++- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/h2o-hadoop-2/assemblyjar.gradle b/h2o-hadoop-2/assemblyjar.gradle index d87a359b6ff9..c9afc98a7348 100644 --- a/h2o-hadoop-2/assemblyjar.gradle +++ b/h2o-hadoop-2/assemblyjar.gradle @@ -19,24 +19,19 @@ if (hasCustomHdfsDep) { } dependencies { - api("org.apache.hadoop:hadoop-client:$hadoopMavenArtifactVersion") { - force = true - } - api("org.apache.hadoop:hadoop-common:$hadoopMavenArtifactVersion") { - force = true - } + api("org.apache.hadoop:hadoop-client") + api("org.apache.hadoop:hadoop-common") + if (hasCustomHdfsDep) { - api("org.apache.hadoop:${hdfsDependency}:$hadoopMavenArtifactVersion") { - force = true - } + api("org.apache.hadoop:${hdfsDependency}") } + api(project(":h2o-mapreduce-generic")) { transitive = false } api project(":h2o-security") api project(":h2o-ext-steam") - // Libraries need for Google Cloud Storage strongly require this Guava version - api('com.google.guava:guava:20.0') {force = true} + api('com.google.guava:guava') api(project(':h2o-app')) { exclude module: "${defaultWebserverModule}" } @@ -78,6 +73,17 @@ dependencies { api(project(":h2o-parquet-parser")) { transitive = false } + + constraints { + api("org.apache.hadoop:hadoop-client:$hadoopMavenArtifactVersion") + api("org.apache.hadoop:hadoop-common:$hadoopMavenArtifactVersion") + if (hasCustomHdfsDep) { + api("org.apache.hadoop:${hdfsDependency}:$hadoopMavenArtifactVersion") + } + // Libraries needed for Google Cloud Storage strongly require this Guava version + api('com.google.guava:guava:20.0') + } + } apply from: "${rootDir}/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle" diff --git a/h2o-hadoop-3/assemblyjar.gradle b/h2o-hadoop-3/assemblyjar.gradle index 56fa74c96463..ddf9f020b8be 100644 --- a/h2o-hadoop-3/assemblyjar.gradle +++ b/h2o-hadoop-3/assemblyjar.gradle @@ -16,10 +16,13 @@ dependencies { api "org.apache.hadoop:hadoop-mapreduce-client-app:$hadoopMavenArtifactVersion" api "org.apache.hadoop:hadoop-aws:$hadoopMavenArtifactVersion" // Libraries need for Google Cloud Storage strongly require this Guava version - api('com.google.guava:guava:20.0') { force = true } + api('com.google.guava:guava') api(project(':h2o-app')) { exclude module: "${defaultWebserverModule}" } + constraints { + api('com.google.guava:guava:20.0') + } } apply from: '../assemblyjar_common.gradle' diff --git a/h2o-hadoop-3/assemblyjar_cdp.gradle b/h2o-hadoop-3/assemblyjar_cdp.gradle index 0f5ca09780e8..41bd314a18be 100644 --- a/h2o-hadoop-3/assemblyjar_cdp.gradle +++ b/h2o-hadoop-3/assemblyjar_cdp.gradle @@ -22,7 +22,7 @@ dependencies { compileOnly "org.apache.hadoop:hadoop-client:$hadoopMavenArtifactVersion" compileOnly "org.apache.hadoop:hadoop-mapreduce-client-app:$hadoopMavenArtifactVersion" // Libraries need for Google Cloud Storage strongly require this Guava version - api('com.google.guava:guava:20.0') { force = true } + api('com.google.guava:guava') api(project(':h2o-app')) { exclude module: "${defaultWebserverModule}" exclude module: "h2o-ext-krbstandalone" // exclude here to avoid pulling all transitive dependencies @@ -30,6 +30,9 @@ dependencies { api(project(':h2o-ext-krbstandalone')) { // include without dependencies transitive = false } + constraints { + api('com.google.guava:guava:20.0') + } } apply from: '../assemblyjar_common.gradle' From 61f4f9b8b7bfc36192334284ae232b2d5942ae11 Mon Sep 17 00:00:00 2001 From: Adam Valenta Date: Wed, 24 Apr 2024 12:56:06 +0200 Subject: [PATCH 06/10] fix java c on windows --- gradle/javaIgnoreSymbolFile.gradle | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gradle/javaIgnoreSymbolFile.gradle b/gradle/javaIgnoreSymbolFile.gradle index c88b8fe1b21e..21690a8b93a8 100644 --- a/gradle/javaIgnoreSymbolFile.gradle +++ b/gradle/javaIgnoreSymbolFile.gradle @@ -1,9 +1,14 @@ +import org.apache.tools.ant.taskdefs.condition.Os + compileJava { // This only works when forking and passing 'javac' as an executable options.compilerArgs << '-XDignore.symbol.file' options.fork = true if (System.env.JAVA_HOME != null) { - options.forkOptions.executable = "${System.env.JAVA_HOME}/bin/javac" + if (Os.isFamily(Os.FAMILY_WINDOWS)) + options.forkOptions.executable = "${System.env.JAVA_HOME}/bin/javac.exe" + else + options.forkOptions.executable = "${System.env.JAVA_HOME}/bin/javac" } else { // JAVA_HOME is not set. Using expected javac location options.forkOptions.executable = "/usr/bin/javac" From 353b14819b7c9a6712210634169c40ebc213dd86 Mon Sep 17 00:00:00 2001 From: krasinski <8573352+krasinski@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:58:56 +0200 Subject: [PATCH 07/10] Revert "remove force = true" This reverts commit d05298bcabe9cd578f8ba0bc0ef98b09b084fe5c. --- h2o-hadoop-2/assemblyjar.gradle | 28 +++++++++++----------------- h2o-hadoop-3/assemblyjar.gradle | 5 +---- h2o-hadoop-3/assemblyjar_cdp.gradle | 5 +---- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/h2o-hadoop-2/assemblyjar.gradle b/h2o-hadoop-2/assemblyjar.gradle index c9afc98a7348..d87a359b6ff9 100644 --- a/h2o-hadoop-2/assemblyjar.gradle +++ b/h2o-hadoop-2/assemblyjar.gradle @@ -19,19 +19,24 @@ if (hasCustomHdfsDep) { } dependencies { - api("org.apache.hadoop:hadoop-client") - api("org.apache.hadoop:hadoop-common") - + api("org.apache.hadoop:hadoop-client:$hadoopMavenArtifactVersion") { + force = true + } + api("org.apache.hadoop:hadoop-common:$hadoopMavenArtifactVersion") { + force = true + } if (hasCustomHdfsDep) { - api("org.apache.hadoop:${hdfsDependency}") + api("org.apache.hadoop:${hdfsDependency}:$hadoopMavenArtifactVersion") { + force = true + } } - api(project(":h2o-mapreduce-generic")) { transitive = false } api project(":h2o-security") api project(":h2o-ext-steam") - api('com.google.guava:guava') + // Libraries need for Google Cloud Storage strongly require this Guava version + api('com.google.guava:guava:20.0') {force = true} api(project(':h2o-app')) { exclude module: "${defaultWebserverModule}" } @@ -73,17 +78,6 @@ dependencies { api(project(":h2o-parquet-parser")) { transitive = false } - - constraints { - api("org.apache.hadoop:hadoop-client:$hadoopMavenArtifactVersion") - api("org.apache.hadoop:hadoop-common:$hadoopMavenArtifactVersion") - if (hasCustomHdfsDep) { - api("org.apache.hadoop:${hdfsDependency}:$hadoopMavenArtifactVersion") - } - // Libraries needed for Google Cloud Storage strongly require this Guava version - api('com.google.guava:guava:20.0') - } - } apply from: "${rootDir}/h2o-parsers/h2o-parquet-parser/parquet_dependencies.gradle" diff --git a/h2o-hadoop-3/assemblyjar.gradle b/h2o-hadoop-3/assemblyjar.gradle index ddf9f020b8be..56fa74c96463 100644 --- a/h2o-hadoop-3/assemblyjar.gradle +++ b/h2o-hadoop-3/assemblyjar.gradle @@ -16,13 +16,10 @@ dependencies { api "org.apache.hadoop:hadoop-mapreduce-client-app:$hadoopMavenArtifactVersion" api "org.apache.hadoop:hadoop-aws:$hadoopMavenArtifactVersion" // Libraries need for Google Cloud Storage strongly require this Guava version - api('com.google.guava:guava') + api('com.google.guava:guava:20.0') { force = true } api(project(':h2o-app')) { exclude module: "${defaultWebserverModule}" } - constraints { - api('com.google.guava:guava:20.0') - } } apply from: '../assemblyjar_common.gradle' diff --git a/h2o-hadoop-3/assemblyjar_cdp.gradle b/h2o-hadoop-3/assemblyjar_cdp.gradle index 41bd314a18be..0f5ca09780e8 100644 --- a/h2o-hadoop-3/assemblyjar_cdp.gradle +++ b/h2o-hadoop-3/assemblyjar_cdp.gradle @@ -22,7 +22,7 @@ dependencies { compileOnly "org.apache.hadoop:hadoop-client:$hadoopMavenArtifactVersion" compileOnly "org.apache.hadoop:hadoop-mapreduce-client-app:$hadoopMavenArtifactVersion" // Libraries need for Google Cloud Storage strongly require this Guava version - api('com.google.guava:guava') + api('com.google.guava:guava:20.0') { force = true } api(project(':h2o-app')) { exclude module: "${defaultWebserverModule}" exclude module: "h2o-ext-krbstandalone" // exclude here to avoid pulling all transitive dependencies @@ -30,9 +30,6 @@ dependencies { api(project(':h2o-ext-krbstandalone')) { // include without dependencies transitive = false } - constraints { - api('com.google.guava:guava:20.0') - } } apply from: '../assemblyjar_common.gradle' From 4dafffc6b74088511dbeaf5c82982e72c7926fe8 Mon Sep 17 00:00:00 2001 From: krasinski <8573352+krasinski@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:05:42 +0200 Subject: [PATCH 08/10] remove force = true v2 --- h2o-hadoop-2/assemblyjar.gradle | 16 +++++++++------- h2o-hadoop-3/assemblyjar.gradle | 4 +++- h2o-hadoop-3/assemblyjar_cdp.gradle | 4 +++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/h2o-hadoop-2/assemblyjar.gradle b/h2o-hadoop-2/assemblyjar.gradle index d87a359b6ff9..dbd5006384e8 100644 --- a/h2o-hadoop-2/assemblyjar.gradle +++ b/h2o-hadoop-2/assemblyjar.gradle @@ -19,15 +19,15 @@ if (hasCustomHdfsDep) { } dependencies { - api("org.apache.hadoop:hadoop-client:$hadoopMavenArtifactVersion") { - force = true + api("org.apache.hadoop:hadoop-client") { + version { strictly hadoopMavenArtifactVersion } } - api("org.apache.hadoop:hadoop-common:$hadoopMavenArtifactVersion") { - force = true + api("org.apache.hadoop:hadoop-common") { + version { strictly hadoopMavenArtifactVersion } } if (hasCustomHdfsDep) { - api("org.apache.hadoop:${hdfsDependency}:$hadoopMavenArtifactVersion") { - force = true + api("org.apache.hadoop:${hdfsDependency}") { + version { strictly hadoopMavenArtifactVersion } } } api(project(":h2o-mapreduce-generic")) { @@ -36,7 +36,9 @@ dependencies { api project(":h2o-security") api project(":h2o-ext-steam") // Libraries need for Google Cloud Storage strongly require this Guava version - api('com.google.guava:guava:20.0') {force = true} + api('com.google.guava:guava') { + version { strictly '20.0' } + } api(project(':h2o-app')) { exclude module: "${defaultWebserverModule}" } diff --git a/h2o-hadoop-3/assemblyjar.gradle b/h2o-hadoop-3/assemblyjar.gradle index 56fa74c96463..a5c66a2a919a 100644 --- a/h2o-hadoop-3/assemblyjar.gradle +++ b/h2o-hadoop-3/assemblyjar.gradle @@ -16,7 +16,9 @@ dependencies { api "org.apache.hadoop:hadoop-mapreduce-client-app:$hadoopMavenArtifactVersion" api "org.apache.hadoop:hadoop-aws:$hadoopMavenArtifactVersion" // Libraries need for Google Cloud Storage strongly require this Guava version - api('com.google.guava:guava:20.0') { force = true } + api('com.google.guava:guava') { + version { strictly '20.0' } + } api(project(':h2o-app')) { exclude module: "${defaultWebserverModule}" } diff --git a/h2o-hadoop-3/assemblyjar_cdp.gradle b/h2o-hadoop-3/assemblyjar_cdp.gradle index 0f5ca09780e8..2785be12c737 100644 --- a/h2o-hadoop-3/assemblyjar_cdp.gradle +++ b/h2o-hadoop-3/assemblyjar_cdp.gradle @@ -22,7 +22,9 @@ dependencies { compileOnly "org.apache.hadoop:hadoop-client:$hadoopMavenArtifactVersion" compileOnly "org.apache.hadoop:hadoop-mapreduce-client-app:$hadoopMavenArtifactVersion" // Libraries need for Google Cloud Storage strongly require this Guava version - api('com.google.guava:guava:20.0') { force = true } + api('com.google.guava:guava') { + version { strictly '20.0' } + } api(project(':h2o-app')) { exclude module: "${defaultWebserverModule}" exclude module: "h2o-ext-krbstandalone" // exclude here to avoid pulling all transitive dependencies From 018b39c1a4c6b75c188b3e11fafd6a01c1af55cb Mon Sep 17 00:00:00 2001 From: Adam Valenta Date: Tue, 28 May 2024 18:14:16 +0200 Subject: [PATCH 09/10] try to fix not deterministic fail on hadoop tests --- h2o-py-cloud-extensions/build.gradle | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/h2o-py-cloud-extensions/build.gradle b/h2o-py-cloud-extensions/build.gradle index b262686fe712..de44cd59c2a9 100644 --- a/h2o-py-cloud-extensions/build.gradle +++ b/h2o-py-cloud-extensions/build.gradle @@ -15,19 +15,7 @@ ext { testsPath = file("tests") } -// -// Create a file with version for Python dist task -// -task createVersionFiles() { - doLast { - file("${buildDir}/h2o_cloud_extensions/").mkdirs() - File version_file = new File("${buildDir}/h2o_cloud_extensions/", "version.txt") - version_file.write(PROJECT_VERSION) - File build_file = new File("${buildDir}/h2o_cloud_extensions/", "buildinfo.txt") - build_file.write(buildVersion.toString()) - } -} task copySrcFiles(type: Copy) { @@ -41,7 +29,13 @@ task copySrcFiles(type: Copy) { into "${buildDir}" } -task buildDist(type: Exec, dependsOn: [createVersionFiles, copySrcFiles]) { +task buildDist(type: Exec, dependsOn: copySrcFiles) { + file("${buildDir}/h2o_cloud_extensions/").mkdirs() + File version_file = new File("${buildDir}/h2o_cloud_extensions/", "version.txt") + version_file.write(PROJECT_VERSION) + + File build_file = new File("${buildDir}/h2o_cloud_extensions/", "buildinfo.txt") + build_file.write(buildVersion.toString()) workingDir buildDir doFirst { file("${buildDir}/tmp").mkdirs() From f6a28466ced6d3371f2c8772dc9cb0e7f01c33fb Mon Sep 17 00:00:00 2001 From: Adam Valenta Date: Thu, 30 May 2024 14:18:42 +0200 Subject: [PATCH 10/10] test 2 --- h2o-py-cloud-extensions/build.gradle | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/h2o-py-cloud-extensions/build.gradle b/h2o-py-cloud-extensions/build.gradle index de44cd59c2a9..cba2e9f8f6c7 100644 --- a/h2o-py-cloud-extensions/build.gradle +++ b/h2o-py-cloud-extensions/build.gradle @@ -30,14 +30,14 @@ task copySrcFiles(type: Copy) { } task buildDist(type: Exec, dependsOn: copySrcFiles) { - file("${buildDir}/h2o_cloud_extensions/").mkdirs() - File version_file = new File("${buildDir}/h2o_cloud_extensions/", "version.txt") - version_file.write(PROJECT_VERSION) - - File build_file = new File("${buildDir}/h2o_cloud_extensions/", "buildinfo.txt") - build_file.write(buildVersion.toString()) workingDir buildDir doFirst { + file("${buildDir}/h2o_cloud_extensions/").mkdirs() + File version_file = new File("${buildDir}/h2o_cloud_extensions/", "version.txt") + version_file.write(PROJECT_VERSION) + + File build_file = new File("${buildDir}/h2o_cloud_extensions/", "buildinfo.txt") + build_file.write(buildVersion.toString()) file("${buildDir}/tmp").mkdirs() standardOutput = new FileOutputStream(file("${buildDir}/tmp/h2o-py_buildDist.out")) }