Prepare for 5.2.0 release

.github/workflows/release.yml

@@ -20,13 +20,13 @@ jobs:
           - nsq
         include:
           - app: kinesis
-            run_snyk: ${{ !contains(github.ref, 'rc') }}
+            run_snyk: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }}
           - app: pubsub
-            run_snyk: ${{ !contains(github.ref, 'rc') }}
+            run_snyk: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }}
           - app: kafka
-            run_snyk: ${{ !contains(github.ref, 'rc') }}
+            run_snyk: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }}
           - app: nsq
-            run_snyk: ${{ !contains(github.ref, 'rc') }}
+            run_snyk: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }}
     steps:
     - uses: actions/checkout@v2
       if: startsWith(github.ref, 'refs/tags/')
@@ -40,6 +40,8 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: 11
+    - name: Install sbt
+      uses: sbt/setup-sbt@v1
     - name: Docker login
       uses: docker/login-action@v1
       with:
@@ -142,6 +144,8 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: 11
+    - name: Install sbt
+      uses: sbt/setup-sbt@v1
     - name: Deploy SCE on Maven Central
       run: sbt "project common" ci-release
       env:

.github/workflows/test.yml

@@ -30,6 +30,8 @@ jobs:
       uses: actions/setup-java@v1
       with:
         java-version: 11
+    - name: Install sbt
+      uses: sbt/setup-sbt@v1
     - name: Start HTTP server to test API enrichment in Common Enrich
       run: python integration-tests/common/api-enrichment-test.py 8001 &
     - name: Start HTTP server to test API enrichment in Common Fs2
@@ -58,16 +60,12 @@ jobs:
     - name: Run integration tests for enrich-kafka
       run: |
         sbt "project kafka" "docker:publishLocal"
-        docker-compose -f integration-tests/enrich-kafka/docker-compose.yml up -d
+        docker compose -f integration-tests/enrich-kafka/docker-compose.yml up -d
         sbt "project kafka" IntegrationTest/test
-        docker-compose -f integration-tests/enrich-kafka/docker-compose.yml down
+        docker compose -f integration-tests/enrich-kafka/docker-compose.yml down
     - name: Run integration tests for enrich-nsq
       run: sbt "project nsqDistroless" IntegrationTest/test
     - name: Generate coverage report
       run: sbt coverageReport
     - name: Aggregate coverage data
       run: sbt coverageAggregate
-    - name: Submit coveralls data
-      run: sbt coveralls
-      env:
-        COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}

CHANGELOG

@@ -1,3 +1,20 @@
+5.2.0 (2024-12-20)
+------------------
+Install sbt in GitHub workflows
+Remove broken CI badge and link in README
+Upgrade dependencies
+Move the defaults for atomic fields lengths and max JSON depth from common-fs2 to common
+Metadata feature re-uses tracker and emits batches
+Improve cpu utilization on large instances
+Check if JSON entity exceeds max depth during conversion from String to Circe JSON
+Pass request headers to the JS enrichment (#901)
+Make maxJsonDepth when validating JSON configurable
+Enrich Kafka: Use 'docker compose' instead of 'docker-compose' in the integration tests
+Remove submit coveralls data step from Github Action workflow
+Enrich Kinesis: Bump Localstack to 3.4.0 in the integration tests
+Enrich Kinesis: remove BeforeAll trait from EnrichKinesisSpec
+Change SLULA license from 1.0 to 1.1
+
 5.0.0 (2024-06-14)
 ------------------
 Add possibilty to emit failed events in TSV format into a third stream (#872)

LICENSE.md

@@ -1,18 +1,20 @@
 # Snowplow Limited Use License Agreement
 
-_Version 1.0, January 2024_
+_Version 1.1, November, 2024_
 
-This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) sets forth the terms on which Snowplow Analytics, Ltd. (“Snowplow”) makes available certain software (the “Software”). BY INSTALLING, DOWNLOADING, ACCESSING, OR USING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or the entity on whose behalf you are receiving the Software.
+This Snowplow Limited Use License Agreement, Version 1.1 (the “Agreement”) sets forth the terms on which Snowplow Analytics, Ltd. (“Snowplow”) makes available certain software (the “Software”). BY INSTALLING, DOWNLOADING, ACCESSING, OR USING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or the entity on whose behalf you are receiving the Software.
 
-## LICENSE GRANT AND CONDITIONS
+## 1. LICENSE GRANT AND CONDITIONS
 
-**1.1 License.** Subject to the terms and conditions of this Agreement, Snowplow hereby grants to Licensee a non-exclusive, royalty-free, worldwide, non-transferable, non-sublicensable license during the term of this Agreement to: (a) use the Software; (b) prepare modifications and derivative works of the Software; and (c) reproduce copies of the Software (the “License”). No right to distribute or make available the Software is granted under this License. Licensee is not granted the right to, and Licensee shall not, exercise the License for any Excluded Purpose.
+**1.1 License.** Subject to the terms and conditions of this Agreement, Snowplow hereby grants to Licensee a non-exclusive, royalty-free, worldwide, non-transferable, non-sublicensable license during the term of this Agreement to: (a) use the Software; (b) prepare modifications and derivative works of the Software; and (c) reproduce copies of the Software (the “License”). No right to distribute or make available the Software is granted under this License. Licensee is not granted the right to, and Licensee shall not, exercise the License for any Competing Use, and Licensee may exercise the License only for Non-Production Use or Non-Commercial Use.
 
-**1.2** For purposes of this Agreement, an “Excluded Purpose” is any use that is either a Competing Use or a Highly-Available Production Use, or both of them.
+**1.2 Definitions.** For purposes of this Agreement:
 
-* **1.2.1** A “Competing Use” is making available any on-premises or distributed software product, or any software-as-a-service, platform-as-a-service, infrastructure-as-a-service, or other similar online service, that competes with any products or services that Snowplow or any of its affiliates provides using the Software.
+* **1.2.1** “Competing Use” is making available any on-premises or distributed software product, or any software-as-a-service, platform-as-a-service, infrastructure-as-a-service, or other similar online service, that competes with any products or services that Snowplow or any of its affiliates provides using the Software.
 
-* **1.2.2** Highly-Available Production Use is any highly-available use, including without limitation any use where multiple instances of any Software component run concurrently to avoid a single point of failure, in a production environment, where production means use on live data.
+* **1.2.2** “Non-Production Use” means any use of the Software to process test or synthetic data to evaluate the sufficiency of the Software for use by Licensee.
+
+* **1.2.3** “Non-Commercial Use” is only: (a) personal use for research, experiment, personal study, or hobby projects, without any anticipated commercial application, or (b) use for teaching purposes by lecturers of a school or university.
 
 **1.3 Conditions.** In consideration of the License, Licensee’s use of the Software is subject to the following conditions:
 
@@ -22,8 +24,8 @@ This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) s
 
   ```
   This software is made available by Snowplow Analytics, Ltd.,
-  under the terms of the Snowplow Limited Use License Agreement, Version 1.0
-  located at https://docs.snowplow.io/limited-use-license-1.0
+  under the terms of the Snowplow Limited Use License Agreement, Version 1.1
+  located at https://docs.snowplow.io/limited-use-license-1.1
   BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
   OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
   ```
@@ -32,23 +34,23 @@ This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) s
 
 **1.5 No Sublicensing.** The License does not include the right to sublicense the Software, however, each recipient to which Licensee provides the Software may exercise the Licenses so long as such recipient agrees to the terms and conditions of this Agreement.
 
-## TERM AND TERMINATION
+## 2. TERM AND TERMINATION
 
 This Agreement will continue unless and until earlier terminated as set forth herein. If Licensee breaches any of its conditions or obligations under this Agreement, this Agreement will terminate automatically and the License will terminate automatically and permanently.
 
-## INTELLECTUAL PROPERTY
+## 3. INTELLECTUAL PROPERTY
 
 As between the parties, Snowplow will retain all right, title, and interest in the Software, and all intellectual property rights therein. Snowplow hereby reserves all rights not expressly granted to Licensee in this Agreement. Snowplow hereby reserves all rights in its trademarks and service marks, and no licenses therein are granted in this Agreement.
 
-## DISCLAIMER
+## 4. DISCLAIMER
 
 SNOWPLOW HEREBY DISCLAIMS ANY AND ALL WARRANTIES AND CONDITIONS, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, AND SPECIFICALLY DISCLAIMS ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, WITH RESPECT TO THE SOFTWARE.
 
-## LIMITATION OF LIABILITY
+## 5. LIMITATION OF LIABILITY
 
 SNOWPLOW WILL NOT BE LIABLE FOR ANY DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO LOST PROFITS OR ANY CONSEQUENTIAL, SPECIAL, INCIDENTAL, INDIRECT, OR DIRECT DAMAGES, HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, ARISING OUT OF THIS AGREEMENT. THE FOREGOING SHALL APPLY TO THE EXTENT PERMITTED BY APPLICABLE LAW.
 
-## GENERAL
+## 6. GENERAL
 
 **6.1 Governing Law.** This Agreement will be governed by and interpreted in accordance with the laws of the state of Delaware, without reference to its conflict of laws principles. If Licensee is located within the United States, all disputes arising out of this Agreement are subject to the exclusive jurisdiction of courts located in Delaware, USA. If Licensee is located outside of the United States, any dispute, controversy or claim arising out of or relating to this Agreement will be referred to and finally determined by arbitration in accordance with the JAMS International Arbitration Rules. The tribunal will consist of one arbitrator. The place of arbitration will be in the State of Delaware, USA. The language to be used in the arbitral proceedings will be English. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction thereof.
 

README.md

@@ -1,7 +1,6 @@
 [![Release][release-image]][releases]
 [![License][license-image]][license]
 [![Coverage Status][coveralls-image]][coveralls]
-[![CI][ci-image]][ci]
 
 # Snowplow Enrich
 
@@ -37,16 +36,13 @@ Licensed under the [Snowplow Limited Use License Agreement][license]. _(If you a
 [roadmap]: https://github.com/snowplow/enrich/issues
 [contributing]: https://docs.snowplowanalytics.com/docs/contributing/
 
-[ci]: https://github.com/snowplow/enrich/actions?query=workflow%3ACI
-[ci-image]: https://github.com/snowplow/enrich/workflows/CI/badge.svg
-
 [coveralls]: https://coveralls.io/github/snowplow/enrich?branch=master
 [coveralls-image]: https://coveralls.io/repos/github/snowplow/enrich/badge.svg?branch=master
 
-[release-image]: https://img.shields.io/badge/release-5.0.0-blue.svg?style=flat
+[release-image]: https://img.shields.io/github/v/release/snowplow/enrich?sort=semver&style=flat
 [releases]: https://github.com/snowplow/enrich/releases
 
-[license]: https://docs.snowplow.io/limited-use-license-1.0
-[license-image]: https://img.shields.io/badge/license-Snowplow--Limited-Use-blue.svg?style=flat
+[license]: https://docs.snowplow.io/limited-use-license-1.1
+[license-image]: https://img.shields.io/badge/license-Snowplow--Limited--Use-blue.svg?style=flat
 
 [faq]: https://docs.snowplow.io/docs/contributing/limited-use-license-faq/

build.sbt

@@ -3,8 +3,8 @@
  * All rights reserved.
  *
  * This software is made available by Snowplow Analytics, Ltd.,
- * under the terms of the Snowplow Limited Use License Agreement, Version 1.0
- * located at https://docs.snowplow.io/limited-use-license-1.0
+ * under the terms of the Snowplow Limited Use License Agreement, Version 1.1
+ * located at https://docs.snowplow.io/limited-use-license-1.1
  * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
  * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
  */

config/config.kafka.extended.hocon

@@ -293,5 +293,9 @@
         "mkt_clickid": 100000
         # ...and any other 'atomic' field with custom limit
     }
-  }  
+  }
+
+  # Optional. The maximum allowed depth for the JSON entities in the events. The default value is 40.
+  # Event will be sent to bad row stream if it contains JSON entity with a depth that exceeds this value.
+  "maxJsonDepth": 50
 }

config/config.kinesis.extended.hocon

@@ -420,4 +420,8 @@
         # ...and any other 'atomic' field with custom limit
     }
   }
+
+  # Optional. The maximum allowed depth for the JSON entities in the events. The default value is 40.
+  # Event will be sent to bad row stream if it contains JSON entity with a depth that exceeds this value.
+  "maxJsonDepth": 50
 }

config/config.nsq.extended.hocon

@@ -298,5 +298,9 @@
         "mkt_clickid": 100000
         # ...and any other 'atomic' field with custom limit
     }
-  }  
+  }
+
+  # Optional. The maximum allowed depth for the JSON entities in the events. The default value is 40.
+  # Event will be sent to bad row stream if it contains JSON entity with a depth that exceeds this value.
+  "maxJsonDepth": 50
 }

config/config.pubsub.extended.hocon

@@ -298,5 +298,9 @@
         "mkt_clickid": 100000
         # ...and any other 'atomic' field with custom limit
     }
-  }  
+  }
+
+  # Optional. The maximum allowed depth for the JSON entities in the events. The default value is 40.
+  # Event will be sent to bad row stream if it contains JSON entity with a depth that exceeds this value.
+  "maxJsonDepth": 50
 }

modules/cloudutils/aws/src/main/scala/com/snowplowanalytics/snowplow/enrich/aws/S3Client.scala

@@ -3,8 +3,8 @@
  * All rights reserved.
  *
  * This software is made available by Snowplow Analytics, Ltd.,
- * under the terms of the Snowplow Limited Use License Agreement, Version 1.0
- * located at https://docs.snowplow.io/limited-use-license-1.0
+ * under the terms of the Snowplow Limited Use License Agreement, Version 1.1
+ * located at https://docs.snowplow.io/limited-use-license-1.1
  * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
  * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
  */

modules/cloudutils/azure/src/main/scala/com/snowplowanalytics/snowplow/enrich/azure/AzureStorageClient.scala

@@ -3,8 +3,8 @@
  * All rights reserved.
  *
  * This software is made available by Snowplow Analytics, Ltd.,
- * under the terms of the Snowplow Limited Use License Agreement, Version 1.0
- * located at https://docs.snowplow.io/limited-use-license-1.0
+ * under the terms of the Snowplow Limited Use License Agreement, Version 1.1
+ * located at https://docs.snowplow.io/limited-use-license-1.1
  * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
  * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
  */

modules/cloudutils/gcp/src/main/scala/com/snowplowanalytics/snowplow/enrich/gcp/GcsClient.scala

@@ -3,8 +3,8 @@
  * All rights reserved.
  *
  * This software is made available by Snowplow Analytics, Ltd.,
- * under the terms of the Snowplow Limited Use License Agreement, Version 1.0
- * located at https://docs.snowplow.io/limited-use-license-1.0
+ * under the terms of the Snowplow Limited Use License Agreement, Version 1.1
+ * located at https://docs.snowplow.io/limited-use-license-1.1
  * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
  * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
  */

modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/CollectorPayloadGen.scala

@@ -3,8 +3,8 @@
  * All rights reserved.
  *
  * This software is made available by Snowplow Analytics, Ltd.,
- * under the terms of the Snowplow Limited Use License Agreement, Version 1.0
- * located at https://docs.snowplow.io/limited-use-license-1.0
+ * under the terms of the Snowplow Limited Use License Agreement, Version 1.1
+ * located at https://docs.snowplow.io/limited-use-license-1.1
  * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
  * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
  */

modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/Utils.scala

@@ -3,8 +3,8 @@
  * All rights reserved.
  *
  * This software is made available by Snowplow Analytics, Ltd.,
- * under the terms of the Snowplow Limited Use License Agreement, Version 1.0
- * located at https://docs.snowplow.io/limited-use-license-1.0
+ * under the terms of the Snowplow Limited Use License Agreement, Version 1.1
+ * located at https://docs.snowplow.io/limited-use-license-1.1
  * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION
  * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT.
  */

modules/common-fs2/src/main/resources/application.conf

@@ -3,92 +3,4 @@
     "accept": false
     "accept": ${?ACCEPT_LIMITED_USE_LICENSE}
   }
-  "validation": {
-    "atomicFieldsLimits": { 
-       "app_id": 255
-       "platform": 255
-       "event": 128
-       "event_id": 36
-       "name_tracker": 128
-       "v_tracker": 100
-       "v_collector": 100
-       "v_etl": 100
-       "user_id": 255
-       "user_ipaddress": 128
-       "user_fingerprint": 128
-       "domain_userid": 128
-       "network_userid": 128
-       "geo_country": 2
-       "geo_region": 3
-       "geo_city": 75
-       "geo_zipcode": 15
-       "geo_region_name": 100
-       "ip_isp": 100
-       "ip_organization": 128
-       "ip_domain": 128
-       "ip_netspeed": 100
-       "page_url": 10000 # Different than the one in atomic schema (4096)
-       "page_title": 2000
-       "page_referrer": 10000 # Different than the one in atomic schema (4096)
-       "page_urlscheme": 16
-       "page_urlhost": 255
-       "page_urlpath": 3000
-       "page_urlquery": 6000
-       "page_urlfragment": 3000
-       "refr_urlscheme": 16
-       "refr_urlhost": 255
-       "refr_urlpath": 6000
-       "refr_urlquery": 6000
-       "refr_urlfragment": 3000
-       "refr_medium": 25
-       "refr_source": 50
-       "refr_term": 255
-       "mkt_clickid": 1000 # Different than the one in atomic schema (128)
-       "mkt_network": 64
-       "mkt_medium": 255
-       "mkt_source": 255
-       "mkt_term": 255
-       "mkt_content": 500
-       "mkt_campaign": 255
-       "se_category": 1000
-       "se_action": 1000
-       "se_label": 4096
-       "se_property": 1000
-       "tr_orderid": 255
-       "tr_affiliation": 255
-       "tr_city": 255
-       "tr_state": 255
-       "tr_country": 255
-       "ti_orderid": 255
-       "ti_sku": 255
-       "ti_name": 255
-       "ti_category": 255
-       "useragent": 1000
-       "br_name": 50
-       "br_family": 50
-       "br_version": 50
-       "br_type": 50
-       "br_renderengine": 50
-       "br_lang": 255
-       "br_colordepth": 12
-       "os_name": 50
-       "os_family": 50
-       "os_manufacturer": 50
-       "os_timezone": 255
-       "dvce_type": 50
-       "doc_charset": 128
-       "tr_currency": 3
-       "ti_currency": 3
-       "base_currency": 3
-       "geo_timezone": 64
-       "etl_tags": 500
-       "refr_domain_userid": 128
-       "domain_sessionid": 128
-       "event_vendor": 1000
-       "event_name": 1000
-       "event_format": 128
-       "event_version": 128
-       "event_fingerprint": 128
-    }
-  }
 }