Skip to content

Commit

Permalink
Fix java v2 precommit
Browse files Browse the repository at this point in the history
  • Loading branch information
damondouglas committed Jan 4, 2025
1 parent e42b922 commit 43399dd
Showing 1 changed file with 95 additions and 32 deletions.
127 changes: 95 additions & 32 deletions runners/google-cloud-dataflow-java/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -275,38 +275,81 @@ def createRunnerV2ValidatesRunnerTest = { Map args ->
}
}

// Push docker images to a container registry for use within tests.
// NB: Tasks which consume docker images from the registry should depend on this
// task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct
// task ordering such that the registry doesn't get cleaned up prior to task completion.
def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") {
def javaVer = getSupportedJavaVersion()
if(project.hasProperty('testJavaVersion')) {
javaVer = "java${project.getProperty('testJavaVersion')}"
}
dependsOn ":sdks:java:container:${javaVer}:docker"
def defaultDockerImageName = containerImageName(
name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk",
root: "apache",
tag: project.sdk_version)
tasks.register('examplesJavaRunnerV2IntegrationTestDistroless', Test.class) {
group = "verification"
dependsOn 'buildAndPushDistrolessContainerImage'
def javaVer = project.findProperty('testJavaVersion')
def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
def tag = project.findProperty('dockerTag')
def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}"
def pipelineOptions = [
"--runner=TestDataflowRunner",
"--project=${gcpProject}",
"--region=${gcpRegion}",
"--tempRoot=${dataflowValidatesTempRoot}",
"--sdkContainerImage=${imageURL}",
"--experiments=use_unified_worker,use_runner_v2",
"--firestoreDb=${firestoreDb}",
]
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(pipelineOptions)

include '**/*IT.class'

maxParallelForks 4
classpath = configurations.examplesJavaIntegrationTest
testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs)
useJUnit { }
}

tasks.register('buildAndPushDistrolessContainerImage', Task.class) {
// Only Java 17 and 21 are supported.
// See https://github.com/GoogleContainerTools/distroless/tree/main/java#image-contents.
def allowed = ["java17", "java21"]
doLast {
def javaVer = project.findProperty('testJavaVersion')
if (!allowed.contains(javaVer)) {
throw new GradleException("testJavaVersion must be one of ${allowed}, got: ${javaVer}")
}
if (!project.hasProperty('dockerTag')) {
throw new GradleException("dockerTag is missing but required")
}
def repository = "us.gcr.io/apache-beam-testing/${System.getenv('USER')}"
def tag = project.findProperty('dockerTag')
def imageURL = "${repository}/beam_${javaVer}_sdk_distroless:${tag}"
exec {
commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerJavaImageName}"
executable 'docker'
workingDir rootDir
args = [
'buildx',
'build',
'-t',
imageURL,
'-f',
'sdks/java/container/Dockerfile-distroless',
"--build-arg=BEAM_BASE=gcr.io/apache-beam-testing/beam-sdk/beam_${javaVer}_sdk",
"--build-arg=DISTROLESS_BASE=gcr.io/distroless/${javaVer}-debian12",
'.'
]
}
exec {
commandLine "gcloud", "docker", "--", "push", "${dockerJavaImageName}"
executable 'docker'
args = ['push', imageURL]
}
}
}

tasks.register("buildAndPushDockerJavaDistrolessContainer") {
// Push docker images to a container registry for use within tests.
// NB: Tasks which consume docker images from the registry should depend on this
// task directly ('dependsOn buildAndPushDockerJavaContainer'). This ensures the correct
// task ordering such that the registry doesn't get cleaned up prior to task completion.
def buildAndPushDockerJavaContainer = tasks.register("buildAndPushDockerJavaContainer") {
def javaVer = getSupportedJavaVersion()
if(project.hasProperty('testJavaVersion')) {
javaVer = "java${project.getProperty('testJavaVersion')}"
}
dependsOn ":sdks:java:container:distroless:${javaVer}:docker"
dependsOn ":sdks:java:container:${javaVer}:docker"
def defaultDockerImageName = containerImageName(
name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk_distroless",
name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk",
root: "apache",
tag: project.sdk_version)
doLast {
Expand Down Expand Up @@ -676,19 +719,6 @@ task examplesJavaRunnerV2PreCommit(type: Test) {
useJUnit { }
}

task examplesJavaDistrolessRunnerV2PreCommit(type: Test) {
group = "Verification"
dependsOn buildAndPushDockerJavaDistrolessContainer
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptions)
include '**/WordCountIT.class'
include '**/WindowedWordCountIT.class'

maxParallelForks 4
classpath = configurations.examplesJavaIntegrationTest
testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs)
useJUnit { }
}

task examplesJavaRunnerV2IntegrationTest(type: Test) {
group = "Verification"
dependsOn buildAndPushDockerJavaContainer
Expand Down Expand Up @@ -802,3 +832,36 @@ task GCSUpload(type: JavaExec) {
args "--stagingLocation=${dataflowUploadTemp}/staging",
"--filesToStage=${testFilesToStage}"
}

def buildAndPushDistrolessDockerJavaContainer = tasks.register("buildAndPushDistrolessDockerJavaContainer") {
def javaVer = getSupportedJavaVersion()
if(project.hasProperty('testJavaVersion')) {
javaVer = "java${project.getProperty('testJavaVersion')}"
}
dependsOn ":sdks:java:container:distroless:${javaVer}:docker"
def defaultDockerImageName = containerImageName(
name: "${project.docker_image_default_repo_prefix}${javaVer}_sdk",
root: "apache",
tag: project.sdk_version)
doLast {
exec {
commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerJavaImageName}"
}
exec {
commandLine "gcloud", "docker", "--", "push", "${dockerJavaImageName}"
}
}
}

task examplesJavaDistrolessRunnerV2PreCommit(type: Test) {
group = "Verification"
dependsOn buildAndPushDistrolessDockerJavaContainer
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(runnerV2PipelineOptions)
include '**/WordCountIT.class'
include '**/WindowedWordCountIT.class'

maxParallelForks 4
classpath = configurations.examplesJavaIntegrationTest
testClassesDirs = files(project(":examples:java").sourceSets.test.output.classesDirs)
useJUnit { }
}

0 comments on commit 43399dd

Please sign in to comment.