MAINT: bump to OS 1.0 and data prepper 1.0.0.0-rc1 (#113)

* MAINT: use OpenSearch 1.0.0-beta1 docker image for testing (#72) * MAINT: use OpenSearch 1.0.0-beta1 docker image for testing (#72) Signed-off-by: qchea <[email protected]> * MAINT: restore traceGroup backward compatibility (#74) Signed-off-by: qchea <[email protected]> * Update issue template with multiple labels (#75) Signed-off-by: Vacha Shah <[email protected]> Co-authored-by: Vacha Shah <[email protected]> * MAINT: bump to OS 1.0 and data prepper 1.0.0-rc1 Signed-off-by: qchea <[email protected]> * MAINT: upgrade setting and API Signed-off-by: qchea <[email protected]> * MAINT: old API but new setting Signed-off-by: qchea <[email protected]>
opensearch-project · Jun 8, 2021 · 87b8dc7 · 87b8dc7
1 parent 05bd430
commit 87b8dc7
Show file tree

Hide file tree

Showing 19 changed files with 129 additions and 74 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: bug
+labels: 'bug, untriaged, Beta'
 assignees: ''
 
 ---

diff --git a/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml b/.github/workflows/data-prepper-trace-analytics-e2e-tests.yml
@@ -24,10 +24,10 @@ jobs:
       with:
         repository: 'opensearch-project/OpenSearch'
         path: OpenSearch
-        ref: '1.0.0-beta1'
+        ref: '1.0'
     - name: Build OpenSearch
       working-directory: ./OpenSearch
-      run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=beta1 -Dbuild.snapshot=false
+      run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=rc1 -Dbuild.snapshot=false
     - name: Checkout Data-Prepper
       uses: actions/checkout@v2
     - name: Grant execute permission for gradlew

diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -24,10 +24,10 @@ jobs:
       with:
         repository: 'opensearch-project/OpenSearch'
         path: OpenSearch
-        ref: '1.0.0-beta1'
+        ref: '1.0'
     - name: Build OpenSearch
       working-directory: ./OpenSearch
-      run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=beta1 -Dbuild.snapshot=false
+      run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=rc1 -Dbuild.snapshot=false
     - name: Checkout Data-Prepper
       uses: actions/checkout@v2
     - name: Grant execute permission for gradlew

diff --git a/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-before-1_13_0-integration-tests.yml
@@ -24,10 +24,10 @@ jobs:
         with:
           repository: 'opensearch-project/OpenSearch'
           path: OpenSearch
-          ref: '1.0.0-beta1'
+          ref: '1.0'
       - name: Build OpenSearch
         working-directory: ./OpenSearch
-        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=beta1 -Dbuild.snapshot=false
+        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=rc1 -Dbuild.snapshot=false
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
       - name: Grant execute permission for gradlew
@@ -40,5 +40,5 @@ jobs:
           sleep 90
       - name: Run ODFE tests
         run: |
-          ./gradlew :data-prepper-plugins:opensearch:test --tests "com.amazon.dataprepper.plugins.sink.opensearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin
-          ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
+          ./gradlew :data-prepper-plugins:opensearch:test --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchTests.testOpenSearchConnection" -Dos.host=https://localhost:9200 -Dos.user=admin -Dos.password=admin
+          ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
diff --git a/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml b/.github/workflows/opensearch-sink-odfe-since-1_13_0-integration-tests.yml
@@ -24,10 +24,10 @@ jobs:
         with:
           repository: 'opensearch-project/OpenSearch'
           path: OpenSearch
-          ref: '1.0.0-beta1'
+          ref: '1.0'
       - name: Build OpenSearch
         working-directory: ./OpenSearch
-        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=beta1 -Dbuild.snapshot=false
+        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=rc1 -Dbuild.snapshot=false
       - name: Checkout Data-Prepper
         uses: actions/checkout@v2
       - name: Grant execute permission for gradlew
@@ -40,5 +40,5 @@ jobs:
           sleep 90
       - name: Run ODFE tests
         run: |
-          ./gradlew :data-prepper-plugins:opensearch:test --tests "com.amazon.dataprepper.plugins.sink.opensearch.ODFETests.testODFEConnection" -Dodfe.host=https://localhost:9200 -Dodfe.user=admin -Dodfe.password=admin
-          ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dodfe=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
+          ./gradlew :data-prepper-plugins:opensearch:test --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchTests.testOpenSearchConnection" -Dos.host=https://localhost:9200 -Dos.user=admin -Dos.password=admin
+          ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
diff --git a/.github/workflows/opensearch-sink-os-integration-tests.yml b/.github/workflows/opensearch-sink-os-integration-tests.yml
@@ -0,0 +1,44 @@
+# This workflow will build a Java project with Gradle
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-gradle
+
+name: Data Prepper OpenSearchSink integration tests with OpenSearch
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  integration_tests:
+    strategy:
+      matrix:
+        java: [14]
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      # TODO: replace local built OpenSearch artifact with the public artifact
+      - name: Checkout OpenSearch
+        uses: actions/checkout@v2
+        with:
+          repository: 'opensearch-project/OpenSearch'
+          path: OpenSearch
+          ref: '1.0'
+      - name: Build OpenSearch
+        working-directory: ./OpenSearch
+        run: ./gradlew publishToMavenLocal -Dbuild.version_qualifier=rc1 -Dbuild.snapshot=false
+      - name: Checkout Data-Prepper
+        uses: actions/checkout@v2
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+      - name: Run OpenSearch docker
+        run: |
+          export version=1.0.0-rc1
+          docker pull opensearchstaging/opensearch:$version
+          docker run -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -d opensearchstaging/opensearch:$version
+          sleep 90
+      - name: Run OpenSearch tests
+        run: |
+          ./gradlew :data-prepper-plugins:opensearch:test --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchTests.testOpenSearchConnection" -Dos.host=https://localhost:9200 -Dos.user=admin -Dos.password=admin
+          ./gradlew :data-prepper-plugins:opensearch:integTest --tests "com.amazon.dataprepper.plugins.sink.opensearch.OpenSearchSinkIT" -Dos=true -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername=docker-cluster -Duser=admin -Dpassword=admin
diff --git a/build-resources.gradle b/build-resources.gradle
@@ -12,7 +12,7 @@
 //preferably try to main the alphabetical order
 ext.versionMap = [
         opentelemetry_proto    : '1.0.1-alpha',
-        opensearch_version: '1.0.0-beta1'
+        opensearch_version: '1.0.0-rc1'
 ]
 
 ext.coreProjects = [project(':data-prepper-api'), project(':data-prepper-core'), project('data-prepper-plugins:common')]
diff --git a/build.gradle b/build.gradle
@@ -12,7 +12,7 @@
 apply from: file("${rootDir}/build-resources.gradle")
 allprojects {
     group = 'com.amazon'
-    version = '1.0.0.0-beta1'
+    version = '1.0.0.0-rc1'
     repositories {
         mavenCentral()
         maven { url "https://jitpack.io" }

diff --git a/data-prepper-core/integrationTest.gradle b/data-prepper-core/integrationTest.gradle
@@ -125,43 +125,43 @@ def removeDataPrepperDockerContainer(final DockerStopContainer stopDataPrepperDo
 }
 
 /**
- * ODFE Docker tasks
+ * OpenSearch Docker tasks
  */
-task pullOdfeDockerImage(type: DockerPullImage) {
-    image = 'amazon/opendistro-for-elasticsearch:1.13.2'
+task pullOpenSearchDockerImage(type: DockerPullImage) {
+    image = 'opensearchstaging/opensearch:1.0.0-rc1'
 }
 
-task createOdfeDockerContainer(type: DockerCreateContainer) {
+task createOpenSearchDockerContainer(type: DockerCreateContainer) {
     dependsOn createDataPrepperNetwork
-    dependsOn pullOdfeDockerImage
-    targetImageId pullOdfeDockerImage.image
+    dependsOn pullOpenSearchDockerImage
+    targetImageId pullOpenSearchDockerImage.image
     containerName = "node-0.example.com"
     hostConfig.portBindings = ['9200:9200', '9600:9600']
     hostConfig.autoRemove = true
     hostConfig.network = createDataPrepperNetwork.getNetworkName()
     envVars = ['discovery.type':'single-node']
 }
 
-task startOdfeDockerContainer(type: DockerStartContainer) {
-    dependsOn createOdfeDockerContainer
-    targetContainerId createOdfeDockerContainer.getContainerId()
+task startOpenSearchDockerContainer(type: DockerStartContainer) {
+    dependsOn createOpenSearchDockerContainer
+    targetContainerId createOpenSearchDockerContainer.getContainerId()
 
     doLast {
         sleep(90*1000)
     }
 }
 
-task stopOdfeDockerContainer(type: DockerStopContainer) {
-    targetContainerId createOdfeDockerContainer.getContainerId()
+task stopOpenSearchDockerContainer(type: DockerStopContainer) {
+    targetContainerId createOpenSearchDockerContainer.getContainerId()
 }
 
 /**
- * End to end test. Spins up ODFE and DataPrepper docker containers, then runs the integ test
+ * End to end test. Spins up OpenSearch and DataPrepper docker containers, then runs the integ test
  * Stops the docker containers when finished
  */
 task rawSpanEndToEndTest(type: Test) {
     dependsOn build
-    dependsOn startOdfeDockerContainer
+    dependsOn startOpenSearchDockerContainer
     def createDataPrepper1Task = createDataPrepperDockerContainer(
             "rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}")
     def createDataPrepper2Task = createDataPrepperDockerContainer(
@@ -170,8 +170,8 @@ task rawSpanEndToEndTest(type: Test) {
     def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
     dependsOn startDataPrepper1Task
     dependsOn startDataPrepper2Task
-    startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer'
-    startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer'
+    startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer'
+    startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer'
     // wait for data-preppers to be ready
     doFirst {
         sleep(10*1000)
@@ -186,7 +186,7 @@ task rawSpanEndToEndTest(type: Test) {
         includeTestsMatching "com.amazon.dataprepper.integration.EndToEndRawSpanTest*"
     }
 
-    finalizedBy stopOdfeDockerContainer
+    finalizedBy stopOpenSearchDockerContainer
     def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
     def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
     finalizedBy stopDataPrepper1Task
@@ -198,7 +198,7 @@ task rawSpanEndToEndTest(type: Test) {
 
 task serviceMapEndToEndTest(type: Test) {
     dependsOn build
-    dependsOn startOdfeDockerContainer
+    dependsOn startOpenSearchDockerContainer
     def createDataPrepper1Task = createDataPrepperDockerContainer(
             "serviceMapDataPrepper1", "data-prepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}")
     def createDataPrepper2Task = createDataPrepperDockerContainer(
@@ -207,8 +207,8 @@ task serviceMapEndToEndTest(type: Test) {
     def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
     dependsOn startDataPrepper1Task
     dependsOn startDataPrepper2Task
-    startDataPrepper1Task.mustRunAfter 'startOdfeDockerContainer'
-    startDataPrepper2Task.mustRunAfter 'startOdfeDockerContainer'
+    startDataPrepper1Task.mustRunAfter 'startOpenSearchDockerContainer'
+    startDataPrepper2Task.mustRunAfter 'startOpenSearchDockerContainer'
     // wait for data-preppers to be ready
     doFirst {
         sleep(10*1000)
@@ -223,7 +223,7 @@ task serviceMapEndToEndTest(type: Test) {
         includeTestsMatching "com.amazon.dataprepper.integration.EndToEndServiceMapTest*"
     }
 
-    finalizedBy stopOdfeDockerContainer
+    finalizedBy stopOpenSearchDockerContainer
     def stopDataPrepper1Task = stopDataPrepperDockerContainer(startDataPrepper1Task as DockerStartContainer)
     def stopDataPrepper2Task = stopDataPrepperDockerContainer(startDataPrepper2Task as DockerStartContainer)
     finalizedBy stopDataPrepper1Task

diff --git a/...e/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java b/...e/src/integrationTest/java/com/amazon/dataprepper/integration/EndToEndServiceMapTest.java
@@ -88,7 +88,7 @@ public void testPipelineEndToEnd() throws IOException, InterruptedException {
 
         // Wait for service map prepper by 2 * window_duration
         Thread.sleep(6000);
-        await().atMost(10, TimeUnit.SECONDS).untilAsserted(
+        await().atMost(20, TimeUnit.SECONDS).untilAsserted(
                 () -> {
                     final List<Map<String, Object>> foundSources = getSourcesFromIndex(restHighLevelClient, SERVICE_MAP_INDEX_NAME);
                     foundSources.forEach(source -> source.remove("hashId"));

diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle
@@ -91,13 +91,13 @@ configurations.all {
 }
 
 test {
-    if (System.getProperty("odfe.host") == null) {
-        exclude '**/ODFETests.class'
+    if (System.getProperty("os.host") == null) {
+        exclude '**/OpenSearchTests.class'
     }
-    systemProperty "odfe.host", System.getProperty("odfe.host")
-    if (System.getProperty("odfe.user") != null) {
-        systemProperty "odfe.user", System.getProperty("odfe.user")
-        systemProperty "odfe.password", System.getProperty("odfe.password")
+    systemProperty "os.host", System.getProperty("os.host")
+    if (System.getProperty("os.user") != null) {
+        systemProperty "os.user", System.getProperty("os.user")
+        systemProperty "os.password", System.getProperty("os.password")
     }
 }
 
@@ -108,7 +108,7 @@ testClusters.integTest {
 integTest {
     systemProperty 'tests.security.manager', 'false'
 
-    systemProperty "odfe", System.getProperty("odfe")
+    systemProperty "os", System.getProperty("os")
     systemProperty "user", System.getProperty("user")
     systemProperty "password", System.getProperty("password")
 }

diff --git a/...ch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/IndexStateManagement.java b/...ch/src/main/java/com/amazon/dataprepper/plugins/sink/opensearch/IndexStateManagement.java
@@ -44,6 +44,7 @@ public static boolean checkISMEnabled(final RestHighLevelClient restHighLevelCli
     public static Optional<String> checkAndCreatePolicy(
             final RestHighLevelClient restHighLevelClient, final String indexType) throws IOException {
         if (indexType.equals(IndexConstants.RAW)) {
+            // TODO: replace with new _opensearch API
             final String endPoint = "/_opendistro/_ism/policies/" + IndexConstants.RAW_ISM_POLICY;
             Request request = createPolicyRequestFromFile(endPoint, IndexConstants.RAW_ISM_FILE_WITH_ISM_TEMPLATE);
             try {

diff --git a/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-span-index-template.json b/data-prepper-plugins/opensearch/src/main/resources/otel-v1-apm-span-index-template.json
@@ -40,11 +40,11 @@
         "type": "keyword"
       },
       "traceGroup": {
+        "ignore_above": 1024,
+        "type": "keyword"
+      },
+      "traceGroupFields": {
         "properties": {
-          "name": {
-            "ignore_above": 1024,
-            "type": "keyword"
-          },
           "endTime": {
             "type": "date_nanos"
           },