From ba0bbe3e13d4d57663ffbcd63c3961d7fcc230dd Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 24 Jun 2024 17:18:56 +0100 Subject: [PATCH] Prepare for 5.2.0 release --- .github/workflows/release.yml | 12 +- .github/workflows/test.yml | 10 +- CHANGELOG | 17 ++ LICENSE.md | 30 ++-- README.md | 10 +- build.sbt | 4 +- config/config.kafka.extended.hocon | 6 +- config/config.kinesis.extended.hocon | 4 + config/config.nsq.extended.hocon | 6 +- config/config.pubsub.extended.hocon | 6 +- .../snowplow/enrich/aws/S3Client.scala | 4 +- .../enrich/azure/AzureStorageClient.scala | 4 +- .../snowplow/enrich/gcp/GcsClient.scala | 4 +- .../common/fs2/test/CollectorPayloadGen.scala | 4 +- .../enrich/common/fs2/test/Utils.scala | 4 +- .../src/main/resources/application.conf | 88 ---------- .../snowplow/enrich/common/fs2/Assets.scala | 4 +- .../enrich/common/fs2/AttributedData.scala | 4 +- .../enrich/common/fs2/CleanCancellation.scala | 4 +- .../snowplow/enrich/common/fs2/Enrich.scala | 152 +++++++++--------- .../enrich/common/fs2/Environment.scala | 29 ++-- .../snowplow/enrich/common/fs2/Run.scala | 4 +- .../enrich/common/fs2/Telemetry.scala | 4 +- .../common/fs2/config/Base64Hocon.scala | 4 +- .../enrich/common/fs2/config/CliConfig.scala | 4 +- .../enrich/common/fs2/config/ConfigFile.scala | 7 +- .../common/fs2/config/ParsedConfigs.scala | 6 +- .../enrich/common/fs2/config/Sentry.scala | 4 +- .../enrich/common/fs2/config/io.scala | 4 +- .../enrich/common/fs2/config/package.scala | 4 +- .../enrich/common/fs2/io/Clients.scala | 4 +- .../enrich/common/fs2/io/FileSink.scala | 4 +- .../enrich/common/fs2/io/FileSystem.scala | 4 +- .../enrich/common/fs2/io/Metrics.scala | 4 +- .../enrich/common/fs2/io/Retries.scala | 4 +- .../enrich/common/fs2/io/Source.scala | 4 +- .../enrich/common/fs2/io/StatsDReporter.scala | 4 +- .../common/fs2/io/experimental/Metadata.scala | 61 ++++--- .../snowplow/enrich/common/fs2/package.scala | 8 +- .../enrich/common/fs2/AssetsSpec.scala | 4 +- .../enrich/common/fs2/EnrichSpec.scala | 136 +++++++++++++--- .../common/fs2/EventGenEtlPipelineSpec.scala | 14 +- .../enrich/common/fs2/PayloadGen.scala | 4 +- .../common/fs2/blackbox/BlackBoxTesting.scala | 14 +- .../adapters/CallrailAdapterSpec.scala | 4 +- .../adapters/GoogleAnalyticsAdapterSpec.scala | 4 +- .../adapters/HubspotAdapterSpec.scala | 4 +- .../adapters/MailchimpAdapterSpec.scala | 4 +- .../adapters/MailgunAdapterSpec.scala | 4 +- .../adapters/MandrillAdapterSpec.scala | 4 +- .../adapters/MarketoAdapterSpec.scala | 4 +- .../blackbox/adapters/OlarkAdapterSpec.scala | 4 +- .../adapters/PagerDutyAdapterSpec.scala | 4 +- .../adapters/PingdomAdapterSpec.scala | 4 +- .../adapters/SendgridAdapterSpec.scala | 4 +- .../adapters/StatusGatorAdapterSpec.scala | 4 +- .../blackbox/adapters/Tp2AdapterSpec.scala | 14 +- .../adapters/UnbounceAdapterSpec.scala | 4 +- .../ApiRequestEnrichmentSpec.scala | 4 +- .../CampaignAttributionEnrichmentSpec.scala | 4 +- ...versionEnrichmentTransactionItemSpec.scala | 4 +- ...yConversionEnrichmentTransactionSpec.scala | 4 +- .../enrichments/IabEnrichmentSpec.scala | 4 +- .../enrichments/IpLookupsEnrichmentSpec.scala | 4 +- .../enrichments/PiiEnrichmentSpec.scala | 4 +- .../RefererParserEnrichmentSpec.scala | 4 +- .../enrichments/SqlQueryEnrichmentSpec.scala | 4 +- .../enrichments/YauaaEnrichmentSpec.scala | 4 +- .../fs2/blackbox/misc/StructEventSpec.scala | 4 +- .../blackbox/misc/TransactionItemSpec.scala | 4 +- .../fs2/blackbox/misc/TransactionSpec.scala | 4 +- .../blackbox/misc/UnstructEventB64Spec.scala | 4 +- .../fs2/blackbox/misc/UnstructEventSpec.scala | 4 +- .../common/fs2/config/Base64HoconSpec.scala | 4 +- .../common/fs2/config/CliConfigSpec.scala | 4 +- .../common/fs2/config/ConfigFileSpec.scala | 4 +- .../common/fs2/config/ParsedConfigsSpec.scala | 7 +- .../ApiRequestEnrichmentSpec.scala | 4 +- .../fs2/enrichments/IabEnrichmentSpec.scala | 4 +- .../fs2/enrichments/YauaaEnrichmentSpec.scala | 4 +- .../enrich/common/fs2/io/MetadataSpec.scala | 9 +- .../enrich/common/fs2/io/SinkSpec.scala | 4 +- .../common/fs2/io/StatsDReporterSpec.scala | 4 +- .../common/fs2/test/AggregatesSpec.scala | 4 +- .../enrich/common/fs2/test/Counter.scala | 4 +- .../enrich/common/fs2/test/HttpServer.scala | 4 +- .../common/fs2/test/TestEnvironment.scala | 12 +- .../enrich/common/fs2/test/package.scala | 4 +- .../common/src/main/resources/reference.conf | 89 ++++++++++ .../common/EtlPipeline.scala | 12 +- .../common/adapters/AdapterRegistry.scala | 11 +- .../common/adapters/AdaptersSchemas.scala | 4 +- .../common/adapters/RawEvent.scala | 4 +- .../common/adapters/registry/Adapter.scala | 7 +- .../adapters/registry/CallrailAdapter.scala | 7 +- .../registry/CloudfrontAccessLogAdapter.scala | 7 +- .../registry/GoogleAnalyticsAdapter.scala | 7 +- .../adapters/registry/HubSpotAdapter.scala | 13 +- .../adapters/registry/IgluAdapter.scala | 35 ++-- .../adapters/registry/MailchimpAdapter.scala | 7 +- .../adapters/registry/MailgunAdapter.scala | 12 +- .../adapters/registry/MandrillAdapter.scala | 13 +- .../adapters/registry/MarketoAdapter.scala | 17 +- .../adapters/registry/OlarkAdapter.scala | 13 +- .../adapters/registry/PagerdutyAdapter.scala | 13 +- .../adapters/registry/PingdomAdapter.scala | 9 +- .../adapters/registry/RemoteAdapter.scala | 14 +- .../adapters/registry/SendgridAdapter.scala | 17 +- .../registry/StatusGatorAdapter.scala | 7 +- .../adapters/registry/UnbounceAdapter.scala | 13 +- .../registry/UrbanAirshipAdapter.scala | 17 +- .../adapters/registry/VeroAdapter.scala | 17 +- .../registry/snowplow/RedirectAdapter.scala | 21 +-- .../registry/snowplow/Tp1Adapter.scala | 7 +- .../registry/snowplow/Tp2Adapter.scala | 14 +- .../common/enrichments/AtomicFields.scala | 4 +- .../AtomicFieldsLengthValidator.scala | 4 +- .../enrichments/ClientEnrichments.scala | 4 +- .../enrichments/EnrichmentManager.scala | 40 +++-- .../enrichments/EnrichmentRegistry.scala | 4 +- .../common/enrichments/EventEnrichments.scala | 4 +- .../common/enrichments/Failure.scala | 4 +- .../common/enrichments/MiscEnrichments.scala | 4 +- .../common/enrichments/SchemaEnrichment.scala | 4 +- .../common/enrichments/Transform.scala | 4 +- .../registry/AnonIpEnrichment.scala | 4 +- .../registry/CachingEvaluator.scala | 4 +- .../CampaignAttributionEnrichment.scala | 4 +- .../registry/CookieExtractorEnrichment.scala | 4 +- .../registry/CrossNavigationEnrichment.scala | 4 +- .../CurrencyConversionEnrichment.scala | 4 +- .../enrichments/registry/EnrichmentConf.scala | 4 +- .../registry/EventFingerprintEnrichment.scala | 4 +- .../HttpHeaderExtractorEnrichment.scala | 4 +- .../enrichments/registry/IabEnrichment.scala | 4 +- .../registry/IpLookupsEnrichment.scala | 4 +- .../registry/JavascriptScriptEnrichment.scala | 25 +-- .../registry/RefererParserEnrichment.scala | 4 +- .../registry/UaParserEnrichment.scala | 4 +- .../registry/UserAgentUtilsEnrichment.scala | 4 +- .../registry/WeatherEnrichment.scala | 4 +- .../registry/YauaaEnrichment.scala | 4 +- .../apirequest/ApiRequestEnrichment.scala | 4 +- .../registry/apirequest/Cache.scala | 4 +- .../registry/apirequest/Errors.scala | 4 +- .../registry/apirequest/HttpApi.scala | 4 +- .../registry/apirequest/Input.scala | 4 +- .../registry/apirequest/Output.scala | 4 +- .../enrichments/registry/enrichments.scala | 4 +- .../enrichments/registry/pii/Mutators.scala | 4 +- .../pii/PiiPseudonymizerEnrichment.scala | 4 +- .../enrichments/registry/pii/package.scala | 4 +- .../registry/pii/serializers.scala | 4 +- .../registry/sqlquery/DbExecutor.scala | 4 +- .../registry/sqlquery/Errors.scala | 4 +- .../enrichments/registry/sqlquery/Input.scala | 4 +- .../registry/sqlquery/JsonOutput.scala | 4 +- .../registry/sqlquery/Output.scala | 4 +- .../enrichments/registry/sqlquery/Rdbms.scala | 4 +- .../sqlquery/SqlQueryEnrichment.scala | 4 +- .../registry/sqlquery/package.scala | 4 +- .../enrichments/web/PageEnrichments.scala | 4 +- .../common/loaders/CollectorPayload.scala | 4 +- .../common/loaders/IpAddressExtractor.scala | 4 +- .../common/loaders/Loader.scala | 4 +- .../common/loaders/ThriftLoader.scala | 31 ++-- .../common/outputs/EnrichedEvent.scala | 4 +- .../common/utils/AtomicError.scala | 4 +- .../common/utils/CirceUtils.scala | 4 +- .../common/utils/ConversionUtils.scala | 4 +- .../common/utils/HttpClient.scala | 4 +- .../common/utils/IgluUtils.scala | 22 +-- .../common/utils/JsonPath.scala | 4 +- .../common/utils/JsonUtils.scala | 30 +++- .../common/utils/MapTransformer.scala | 4 +- .../common/utils/ShiftExecution.scala | 4 +- .../package.scala | 4 +- .../enrich/CirceJsonDeserializer.scala | 6 +- .../jackson/enrich/CirceJsonModule.scala | 4 +- .../io/circe/jackson/enrich/package.scala | 4 +- .../AcceptInvalid.scala | 4 +- .../EtlPipelineSpec.scala | 18 ++- .../SpecHelpers.scala | 20 ++- .../adapters/registry/AdapterSpec.scala | 7 +- .../registry/CallrailAdapterSpec.scala | 8 +- .../CloudfrontAccessLogAdapterSpec.scala | 50 ++++-- .../registry/GoogleAnalyticsAdapterSpec.scala | 34 ++-- .../registry/HubSpotAdapterSpec.scala | 20 +-- .../adapters/registry/IgluAdapterSpec.scala | 46 +++--- .../registry/MailchimpAdapterSpec.scala | 22 +-- .../registry/MailgunAdapterSpec.scala | 54 +++++-- .../registry/MandrillAdapterSpec.scala | 24 +-- .../registry/MarketoAdapterSpec.scala | 10 +- .../adapters/registry/OlarkAdapterSpec.scala | 28 ++-- .../registry/PagerdutyAdapterSpec.scala | 20 +-- .../registry/PingdomAdapterSpec.scala | 10 +- .../adapters/registry/RemoteAdapterSpec.scala | 16 +- .../registry/SendgridAdapterSpec.scala | 41 +++-- .../registry/StatusGatorAdapterSpec.scala | 22 ++- .../registry/UnbounceAdapterSpec.scala | 46 ++++-- .../registry/UrbanAirshipAdapterSpec.scala | 21 ++- .../adapters/registry/VeroAdapterSpec.scala | 40 +++-- .../snowplow/SnowplowAdapterSpec.scala | 44 ++--- .../enrichments/AtomicFieldsSpec.scala | 4 +- .../enrichments/CachingEvaluatorSpec.scala | 4 +- .../enrichments/EnrichmentManagerSpec.scala | 100 ++++++++---- .../enrichments/EnrichmentRegistrySpec.scala | 4 +- .../enrichments/FailureSpec.scala | 4 +- .../enrichments/SchemaEnrichmentSpec.scala | 4 +- .../enrichments/clientEnrichmentSpecs.scala | 4 +- .../enrichments/eventEnrichmentSpecs.scala | 4 +- .../enrichments/miscEnrichmentSpecs.scala | 4 +- .../registry/AnonIpEnrichmentSpec.scala | 4 +- .../CampaignAttributionEnrichmentSpec.scala | 4 +- .../CookieExtractorEnrichmentSpec.scala | 4 +- .../CrossNavigationEnrichmentSpec.scala | 4 +- .../CurrencyConversionEnrichmentSpec.scala | 4 +- .../registry/EnrichmentConfigsSpec.scala | 4 +- .../EventFingerprintEnrichmentSpec.scala | 4 +- .../HttpHeaderExtractorEnrichmentSpec.scala | 4 +- .../registry/IabEnrichmentSpec.scala | 4 +- .../registry/IpLookupsEnrichmentSpec.scala | 4 +- .../JavascriptScriptEnrichmentSpec.scala | 65 +++++--- .../RefererParserEnrichmentSpec.scala | 4 +- .../registry/UaParserEnrichmentSpec.scala | 4 +- .../UserAgentUtilsEnrichmentSpec.scala | 4 +- .../registry/WeatherEnrichmentSpec.scala | 4 +- .../registry/YauaaEnrichmentSpec.scala | 4 +- .../ApiRequestEnrichmentIntegrationTest.scala | 4 +- .../apirequest/ApiRequestEnrichmentSpec.scala | 4 +- .../registry/apirequest/HttpApiSpec.scala | 4 +- .../registry/apirequest/HttpClientSpec.scala | 4 +- .../registry/apirequest/InputSpec.scala | 4 +- .../registry/apirequest/OutputSpec.scala | 4 +- .../registry/apirequest/ValidatorSpec.scala | 4 +- .../pii/PiiPseudonymizerEnrichmentSpec.scala | 9 +- .../registry/sqlquery/InputSpec.scala | 4 +- .../registry/sqlquery/OutputSpec.scala | 4 +- .../SqlQueryEnrichmentIntegrationTest.scala | 4 +- .../sqlquery/SqlQueryEnrichmentSpec.scala | 4 +- .../enrichments/web/ExtractPageUriSpec.scala | 4 +- .../web/ParseCrossDomainSpec.scala | 4 +- .../loaders/CollectorPayloadSpec.scala | 4 +- .../loaders/IpAddressExtractorSpec.scala | 4 +- .../loaders/LoaderSpec.scala | 4 +- .../loaders/ThriftLoaderSpec.scala | 4 +- .../outputs/EnrichedEventSpec.scala | 4 +- .../utils/CirceUtilsSpec.scala | 4 +- .../utils/IgluUtilsSpec.scala | 152 ++++++++++++------ .../utils/JsonPathSpec.scala | 4 +- .../utils/JsonUtilsSpec.scala | 64 +++++++- .../utils/MapTransformerSpec.scala | 4 +- .../utils/conversionUtilsSpecs.scala | 4 +- .../enrich/kafka/test/EnrichKafkaSpec.scala | 4 +- .../AzureAuthenticationCallbackHandler.scala | 4 +- .../Main.scala | 4 +- .../Sink.scala | 4 +- .../Source.scala | 4 +- .../snowplow/enrich/kafka/ConfigSpec.scala | 10 +- .../snowplow/enrich/kinesis/Containers.scala | 6 +- .../enrich/kinesis/EnrichKinesisSpec.scala | 7 +- .../enrich/kinesis/KinesisConfig.scala | 4 +- .../kinesis/enrichments/ApiRequest.scala | 4 +- .../kinesis/enrichments/Enrichment.scala | 4 +- .../kinesis/enrichments/Enrichments.scala | 4 +- .../kinesis/enrichments/Javascript.scala | 4 +- .../enrich/kinesis/enrichments/SqlQuery.scala | 4 +- .../enrich/kinesis/enrichments/Yauaa.scala | 4 +- .../snowplow/enrich/kinesis/utils.scala | 4 +- .../enrich/kinesis/DynamoDbConfig.scala | 4 +- .../snowplow/enrich/kinesis/Main.scala | 4 +- .../snowplow/enrich/kinesis/Sink.scala | 4 +- .../snowplow/enrich/kinesis/Source.scala | 4 +- .../snowplow/enrich/kinesis/package.scala | 4 +- .../snowplow/enrich/kinesis/ConfigSpec.scala | 10 +- .../snowplow/enrich/kinesis/SinkSpec.scala | 4 +- .../snowplow/enrich/nsq/test/Containers.scala | 4 +- .../enrich/nsq/test/EnrichNsqSpec.scala | 4 +- .../snowplow/enrich/nsq/test/Utils.scala | 4 +- .../snowplow/enrich/nsq/Main.scala | 4 +- .../snowplow/enrich/nsq/Record.scala | 4 +- .../snowplow/enrich/nsq/Sink.scala | 4 +- .../snowplow/enrich/nsq/Source.scala | 4 +- .../snowplow/enrich/nsq/ConfigSpec.scala | 10 +- .../snowplow/enrich/pubsub/Main.scala | 4 +- .../snowplow/enrich/pubsub/Sink.scala | 4 +- .../snowplow/enrich/pubsub/Source.scala | 4 +- .../snowplow/enrich/pubsub/Utils.scala | 4 +- .../snowplow/enrich/pubsub/package.scala | 4 +- .../snowplow/enrich/pubsub/ConfigSpec.scala | 10 +- .../enrich/pubsub/GcpUserAgentSpec.scala | 4 +- project/BuildSettings.scala | 6 +- project/Dependencies.scala | 39 +++-- 293 files changed, 1856 insertions(+), 1260 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 23eb9f52f..9791b70d3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,13 +20,13 @@ jobs: - nsq include: - app: kinesis - run_snyk: ${{ !contains(github.ref, 'rc') }} + run_snyk: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }} - app: pubsub - run_snyk: ${{ !contains(github.ref, 'rc') }} + run_snyk: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }} - app: kafka - run_snyk: ${{ !contains(github.ref, 'rc') }} + run_snyk: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }} - app: nsq - run_snyk: ${{ !contains(github.ref, 'rc') }} + run_snyk: ${{ startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') }} steps: - uses: actions/checkout@v2 if: startsWith(github.ref, 'refs/tags/') @@ -40,6 +40,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: Install sbt + uses: sbt/setup-sbt@v1 - name: Docker login uses: docker/login-action@v1 with: @@ -142,6 +144,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: Install sbt + uses: sbt/setup-sbt@v1 - name: Deploy SCE on Maven Central run: sbt "project common" ci-release env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 42fd5ed22..8a566a6ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,6 +30,8 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: Install sbt + uses: sbt/setup-sbt@v1 - name: Start HTTP server to test API enrichment in Common Enrich run: python integration-tests/common/api-enrichment-test.py 8001 & - name: Start HTTP server to test API enrichment in Common Fs2 @@ -58,16 +60,12 @@ jobs: - name: Run integration tests for enrich-kafka run: | sbt "project kafka" "docker:publishLocal" - docker-compose -f integration-tests/enrich-kafka/docker-compose.yml up -d + docker compose -f integration-tests/enrich-kafka/docker-compose.yml up -d sbt "project kafka" IntegrationTest/test - docker-compose -f integration-tests/enrich-kafka/docker-compose.yml down + docker compose -f integration-tests/enrich-kafka/docker-compose.yml down - name: Run integration tests for enrich-nsq run: sbt "project nsqDistroless" IntegrationTest/test - name: Generate coverage report run: sbt coverageReport - name: Aggregate coverage data run: sbt coverageAggregate - - name: Submit coveralls data - run: sbt coveralls - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG index 0b67beb64..b05da843f 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,20 @@ +5.2.0 (2024-12-20) +------------------ +Install sbt in GitHub workflows +Remove broken CI badge and link in README +Upgrade dependencies +Move the defaults for atomic fields lengths and max JSON depth from common-fs2 to common +Metadata feature re-uses tracker and emits batches +Improve cpu utilization on large instances +Check if JSON entity exceeds max depth during conversion from String to Circe JSON +Pass request headers to the JS enrichment (#901) +Make maxJsonDepth when validating JSON configurable +Enrich Kafka: Use 'docker compose' instead of 'docker-compose' in the integration tests +Remove submit coveralls data step from Github Action workflow +Enrich Kinesis: Bump Localstack to 3.4.0 in the integration tests +Enrich Kinesis: remove BeforeAll trait from EnrichKinesisSpec +Change SLULA license from 1.0 to 1.1 + 5.0.0 (2024-06-14) ------------------ Add possibilty to emit failed events in TSV format into a third stream (#872) diff --git a/LICENSE.md b/LICENSE.md index 6abbe69c7..0a7d3cc95 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,18 +1,20 @@ # Snowplow Limited Use License Agreement -_Version 1.0, January 2024_ +_Version 1.1, November, 2024_ -This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) sets forth the terms on which Snowplow Analytics, Ltd. (“Snowplow”) makes available certain software (the “Software”). BY INSTALLING, DOWNLOADING, ACCESSING, OR USING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or the entity on whose behalf you are receiving the Software. +This Snowplow Limited Use License Agreement, Version 1.1 (the “Agreement”) sets forth the terms on which Snowplow Analytics, Ltd. (“Snowplow”) makes available certain software (the “Software”). BY INSTALLING, DOWNLOADING, ACCESSING, OR USING ANY OF THE SOFTWARE, YOU AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE TO SUCH TERMS AND CONDITIONS, YOU MUST NOT USE THE SOFTWARE. IF YOU ARE RECEIVING THE SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE ACTUAL AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS AGREEMENT ON BEHALF OF SUCH ENTITY. “Licensee” means you, an individual, or the entity on whose behalf you are receiving the Software. -## LICENSE GRANT AND CONDITIONS +## 1. LICENSE GRANT AND CONDITIONS -**1.1 License.** Subject to the terms and conditions of this Agreement, Snowplow hereby grants to Licensee a non-exclusive, royalty-free, worldwide, non-transferable, non-sublicensable license during the term of this Agreement to: (a) use the Software; (b) prepare modifications and derivative works of the Software; and (c) reproduce copies of the Software (the “License”). No right to distribute or make available the Software is granted under this License. Licensee is not granted the right to, and Licensee shall not, exercise the License for any Excluded Purpose. +**1.1 License.** Subject to the terms and conditions of this Agreement, Snowplow hereby grants to Licensee a non-exclusive, royalty-free, worldwide, non-transferable, non-sublicensable license during the term of this Agreement to: (a) use the Software; (b) prepare modifications and derivative works of the Software; and (c) reproduce copies of the Software (the “License”). No right to distribute or make available the Software is granted under this License. Licensee is not granted the right to, and Licensee shall not, exercise the License for any Competing Use, and Licensee may exercise the License only for Non-Production Use or Non-Commercial Use. -**1.2** For purposes of this Agreement, an “Excluded Purpose” is any use that is either a Competing Use or a Highly-Available Production Use, or both of them. +**1.2 Definitions.** For purposes of this Agreement: -* **1.2.1** A “Competing Use” is making available any on-premises or distributed software product, or any software-as-a-service, platform-as-a-service, infrastructure-as-a-service, or other similar online service, that competes with any products or services that Snowplow or any of its affiliates provides using the Software. +* **1.2.1** “Competing Use” is making available any on-premises or distributed software product, or any software-as-a-service, platform-as-a-service, infrastructure-as-a-service, or other similar online service, that competes with any products or services that Snowplow or any of its affiliates provides using the Software. -* **1.2.2** Highly-Available Production Use is any highly-available use, including without limitation any use where multiple instances of any Software component run concurrently to avoid a single point of failure, in a production environment, where production means use on live data. +* **1.2.2** “Non-Production Use” means any use of the Software to process test or synthetic data to evaluate the sufficiency of the Software for use by Licensee. + +* **1.2.3** “Non-Commercial Use” is only: (a) personal use for research, experiment, personal study, or hobby projects, without any anticipated commercial application, or (b) use for teaching purposes by lecturers of a school or university. **1.3 Conditions.** In consideration of the License, Licensee’s use of the Software is subject to the following conditions: @@ -22,8 +24,8 @@ This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) s ``` This software is made available by Snowplow Analytics, Ltd., - under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - located at https://docs.snowplow.io/limited-use-license-1.0 + under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + located at https://docs.snowplow.io/limited-use-license-1.1 BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. ``` @@ -32,23 +34,23 @@ This Snowplow Limited Use License Agreement, Version 1.0 (the “Agreement”) s **1.5 No Sublicensing.** The License does not include the right to sublicense the Software, however, each recipient to which Licensee provides the Software may exercise the Licenses so long as such recipient agrees to the terms and conditions of this Agreement. -## TERM AND TERMINATION +## 2. TERM AND TERMINATION This Agreement will continue unless and until earlier terminated as set forth herein. If Licensee breaches any of its conditions or obligations under this Agreement, this Agreement will terminate automatically and the License will terminate automatically and permanently. -## INTELLECTUAL PROPERTY +## 3. INTELLECTUAL PROPERTY As between the parties, Snowplow will retain all right, title, and interest in the Software, and all intellectual property rights therein. Snowplow hereby reserves all rights not expressly granted to Licensee in this Agreement. Snowplow hereby reserves all rights in its trademarks and service marks, and no licenses therein are granted in this Agreement. -## DISCLAIMER +## 4. DISCLAIMER SNOWPLOW HEREBY DISCLAIMS ANY AND ALL WARRANTIES AND CONDITIONS, EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, AND SPECIFICALLY DISCLAIMS ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, WITH RESPECT TO THE SOFTWARE. -## LIMITATION OF LIABILITY +## 5. LIMITATION OF LIABILITY SNOWPLOW WILL NOT BE LIABLE FOR ANY DAMAGES OF ANY KIND, INCLUDING BUT NOT LIMITED TO LOST PROFITS OR ANY CONSEQUENTIAL, SPECIAL, INCIDENTAL, INDIRECT, OR DIRECT DAMAGES, HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, ARISING OUT OF THIS AGREEMENT. THE FOREGOING SHALL APPLY TO THE EXTENT PERMITTED BY APPLICABLE LAW. -## GENERAL +## 6. GENERAL **6.1 Governing Law.** This Agreement will be governed by and interpreted in accordance with the laws of the state of Delaware, without reference to its conflict of laws principles. If Licensee is located within the United States, all disputes arising out of this Agreement are subject to the exclusive jurisdiction of courts located in Delaware, USA. If Licensee is located outside of the United States, any dispute, controversy or claim arising out of or relating to this Agreement will be referred to and finally determined by arbitration in accordance with the JAMS International Arbitration Rules. The tribunal will consist of one arbitrator. The place of arbitration will be in the State of Delaware, USA. The language to be used in the arbitral proceedings will be English. Judgment upon the award rendered by the arbitrator may be entered in any court having jurisdiction thereof. diff --git a/README.md b/README.md index bd26fd2b6..7093ee12b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ [![Release][release-image]][releases] [![License][license-image]][license] [![Coverage Status][coveralls-image]][coveralls] -[![CI][ci-image]][ci] # Snowplow Enrich @@ -37,16 +36,13 @@ Licensed under the [Snowplow Limited Use License Agreement][license]. _(If you a [roadmap]: https://github.com/snowplow/enrich/issues [contributing]: https://docs.snowplowanalytics.com/docs/contributing/ -[ci]: https://github.com/snowplow/enrich/actions?query=workflow%3ACI -[ci-image]: https://github.com/snowplow/enrich/workflows/CI/badge.svg - [coveralls]: https://coveralls.io/github/snowplow/enrich?branch=master [coveralls-image]: https://coveralls.io/repos/github/snowplow/enrich/badge.svg?branch=master -[release-image]: https://img.shields.io/badge/release-5.0.0-blue.svg?style=flat +[release-image]: https://img.shields.io/github/v/release/snowplow/enrich?sort=semver&style=flat [releases]: https://github.com/snowplow/enrich/releases -[license]: https://docs.snowplow.io/limited-use-license-1.0 -[license-image]: https://img.shields.io/badge/license-Snowplow--Limited-Use-blue.svg?style=flat +[license]: https://docs.snowplow.io/limited-use-license-1.1 +[license-image]: https://img.shields.io/badge/license-Snowplow--Limited--Use-blue.svg?style=flat [faq]: https://docs.snowplow.io/docs/contributing/limited-use-license-faq/ diff --git a/build.sbt b/build.sbt index 324fc296c..68daa7f15 100644 --- a/build.sbt +++ b/build.sbt @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/config/config.kafka.extended.hocon b/config/config.kafka.extended.hocon index 6929018f9..12875c0ef 100644 --- a/config/config.kafka.extended.hocon +++ b/config/config.kafka.extended.hocon @@ -293,5 +293,9 @@ "mkt_clickid": 100000 # ...and any other 'atomic' field with custom limit } - } + } + + # Optional. The maximum allowed depth for the JSON entities in the events. The default value is 40. + # Event will be sent to bad row stream if it contains JSON entity with a depth that exceeds this value. + "maxJsonDepth": 50 } diff --git a/config/config.kinesis.extended.hocon b/config/config.kinesis.extended.hocon index c7f336a96..de09f22b4 100644 --- a/config/config.kinesis.extended.hocon +++ b/config/config.kinesis.extended.hocon @@ -420,4 +420,8 @@ # ...and any other 'atomic' field with custom limit } } + + # Optional. The maximum allowed depth for the JSON entities in the events. The default value is 40. + # Event will be sent to bad row stream if it contains JSON entity with a depth that exceeds this value. + "maxJsonDepth": 50 } diff --git a/config/config.nsq.extended.hocon b/config/config.nsq.extended.hocon index 6e5aae5bd..8615421d0 100644 --- a/config/config.nsq.extended.hocon +++ b/config/config.nsq.extended.hocon @@ -298,5 +298,9 @@ "mkt_clickid": 100000 # ...and any other 'atomic' field with custom limit } - } + } + + # Optional. The maximum allowed depth for the JSON entities in the events. The default value is 40. + # Event will be sent to bad row stream if it contains JSON entity with a depth that exceeds this value. + "maxJsonDepth": 50 } diff --git a/config/config.pubsub.extended.hocon b/config/config.pubsub.extended.hocon index 32425f39d..52b60e984 100644 --- a/config/config.pubsub.extended.hocon +++ b/config/config.pubsub.extended.hocon @@ -298,5 +298,9 @@ "mkt_clickid": 100000 # ...and any other 'atomic' field with custom limit } - } + } + + # Optional. The maximum allowed depth for the JSON entities in the events. The default value is 40. + # Event will be sent to bad row stream if it contains JSON entity with a depth that exceeds this value. + "maxJsonDepth": 50 } diff --git a/modules/cloudutils/aws/src/main/scala/com/snowplowanalytics/snowplow/enrich/aws/S3Client.scala b/modules/cloudutils/aws/src/main/scala/com/snowplowanalytics/snowplow/enrich/aws/S3Client.scala index d5c7da502..aa076f135 100644 --- a/modules/cloudutils/aws/src/main/scala/com/snowplowanalytics/snowplow/enrich/aws/S3Client.scala +++ b/modules/cloudutils/aws/src/main/scala/com/snowplowanalytics/snowplow/enrich/aws/S3Client.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/cloudutils/azure/src/main/scala/com/snowplowanalytics/snowplow/enrich/azure/AzureStorageClient.scala b/modules/cloudutils/azure/src/main/scala/com/snowplowanalytics/snowplow/enrich/azure/AzureStorageClient.scala index eca47c073..250fbd8c9 100644 --- a/modules/cloudutils/azure/src/main/scala/com/snowplowanalytics/snowplow/enrich/azure/AzureStorageClient.scala +++ b/modules/cloudutils/azure/src/main/scala/com/snowplowanalytics/snowplow/enrich/azure/AzureStorageClient.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/cloudutils/gcp/src/main/scala/com/snowplowanalytics/snowplow/enrich/gcp/GcsClient.scala b/modules/cloudutils/gcp/src/main/scala/com/snowplowanalytics/snowplow/enrich/gcp/GcsClient.scala index c0c87d16d..065f3457e 100644 --- a/modules/cloudutils/gcp/src/main/scala/com/snowplowanalytics/snowplow/enrich/gcp/GcsClient.scala +++ b/modules/cloudutils/gcp/src/main/scala/com/snowplowanalytics/snowplow/enrich/gcp/GcsClient.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/CollectorPayloadGen.scala b/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/CollectorPayloadGen.scala index f8c9157d5..9aa112e2b 100644 --- a/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/CollectorPayloadGen.scala +++ b/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/CollectorPayloadGen.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/Utils.scala b/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/Utils.scala index 7377f4850..d645357f6 100644 --- a/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/Utils.scala +++ b/modules/common-fs2/src/it/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/Utils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/resources/application.conf b/modules/common-fs2/src/main/resources/application.conf index 226861048..fdf628816 100644 --- a/modules/common-fs2/src/main/resources/application.conf +++ b/modules/common-fs2/src/main/resources/application.conf @@ -3,92 +3,4 @@ "accept": false "accept": ${?ACCEPT_LIMITED_USE_LICENSE} } - "validation": { - "atomicFieldsLimits": { - "app_id": 255 - "platform": 255 - "event": 128 - "event_id": 36 - "name_tracker": 128 - "v_tracker": 100 - "v_collector": 100 - "v_etl": 100 - "user_id": 255 - "user_ipaddress": 128 - "user_fingerprint": 128 - "domain_userid": 128 - "network_userid": 128 - "geo_country": 2 - "geo_region": 3 - "geo_city": 75 - "geo_zipcode": 15 - "geo_region_name": 100 - "ip_isp": 100 - "ip_organization": 128 - "ip_domain": 128 - "ip_netspeed": 100 - "page_url": 10000 # Different than the one in atomic schema (4096) - "page_title": 2000 - "page_referrer": 10000 # Different than the one in atomic schema (4096) - "page_urlscheme": 16 - "page_urlhost": 255 - "page_urlpath": 3000 - "page_urlquery": 6000 - "page_urlfragment": 3000 - "refr_urlscheme": 16 - "refr_urlhost": 255 - "refr_urlpath": 6000 - "refr_urlquery": 6000 - "refr_urlfragment": 3000 - "refr_medium": 25 - "refr_source": 50 - "refr_term": 255 - "mkt_clickid": 1000 # Different than the one in atomic schema (128) - "mkt_network": 64 - "mkt_medium": 255 - "mkt_source": 255 - "mkt_term": 255 - "mkt_content": 500 - "mkt_campaign": 255 - "se_category": 1000 - "se_action": 1000 - "se_label": 4096 - "se_property": 1000 - "tr_orderid": 255 - "tr_affiliation": 255 - "tr_city": 255 - "tr_state": 255 - "tr_country": 255 - "ti_orderid": 255 - "ti_sku": 255 - "ti_name": 255 - "ti_category": 255 - "useragent": 1000 - "br_name": 50 - "br_family": 50 - "br_version": 50 - "br_type": 50 - "br_renderengine": 50 - "br_lang": 255 - "br_colordepth": 12 - "os_name": 50 - "os_family": 50 - "os_manufacturer": 50 - "os_timezone": 255 - "dvce_type": 50 - "doc_charset": 128 - "tr_currency": 3 - "ti_currency": 3 - "base_currency": 3 - "geo_timezone": 64 - "etl_tags": 500 - "refr_domain_userid": 128 - "domain_sessionid": 128 - "event_vendor": 1000 - "event_name": 1000 - "event_format": 128 - "event_version": 128 - "event_fingerprint": 128 - } - } } diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala index 90dba7903..c00bfb6d1 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Assets.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/AttributedData.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/AttributedData.scala index 9356c2f48..64642de79 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/AttributedData.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/AttributedData.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/CleanCancellation.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/CleanCancellation.scala index a3142f919..9ddfa64f5 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/CleanCancellation.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/CleanCancellation.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Enrich.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Enrich.scala index 9984e5b3d..895053cd5 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Enrich.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Enrich.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -20,10 +20,10 @@ import cats.data.{Ior, NonEmptyList, ValidatedNel} import cats.{Monad, Parallel} import cats.implicits._ -import cats.effect.kernel.{Async, Clock, Sync} +import cats.effect.kernel.{Async, Sync} import cats.effect.implicits._ -import fs2.{Pipe, Stream} +import fs2.{Chunk, Pipe, Stream} import _root_.io.sentry.SentryClient @@ -51,6 +51,12 @@ object Enrich { private implicit def unsafeLogger[F[_]: Sync]: Logger[F] = Slf4jLogger.getLogger[F] + case class Result[+A]( + original: A, + enriched: List[Enriched], + collectorTstamp: Option[Long] + ) + /** * Run a primary enrichment stream, reading from [[Environment]] source, enriching * via [[enrichWith]] and sinking into the Good, Bad, and Pii sinks. @@ -61,58 +67,58 @@ object Enrich { * they'll be refreshed periodically by [[Assets.updateStream]] */ def run[F[_]: Async, A](env: Environment[F, A]): Stream[F, Unit] = { - val enrichmentsRegistry: F[EnrichmentRegistry[F]] = env.enrichments.get.map(_.registry) - val enrich: Enrich[F] = - enrichWith[F]( - enrichmentsRegistry, - env.adapterRegistry, - env.igluClient, - env.sentry, - env.processor, - env.featureFlags, - env.metrics.invalidCount, - env.registryLookup, - env.atomicFields, - env.sinkIncomplete.isDefined - ) - - val enriched = - env.source.chunks - .evalTap(chunk => Logger[F].debug(s"Starting to process chunk of size ${chunk.size}")) - .evalTap(chunk => env.metrics.rawCount(chunk.size)) - .map(chunk => chunk.map(a => (a, env.getPayload(a)))) - .evalMap(chunk => - for { - begin <- Clock[F].realTime - result <- - env.semaphore.permit.use { _ => - chunk.toList.map { case (orig, bytes) => enrich(bytes).map((orig, _)) }.parSequenceN(env.streamsSettings.concurrency.enrich) - } - end <- Clock[F].realTime - _ <- Logger[F].debug(s"Chunk of size ${chunk.size} enriched in ${end - begin} ms") - } yield result - ) - val sinkAndCheckpoint: Pipe[F, List[(A, Result)], Unit] = - _.parEvalMap(env.streamsSettings.concurrency.sink)(chunk => + def enrichChunk(chunk: Chunk[A]): F[List[Result[A]]] = + env.semaphore.permit.surround { + for { + _ <- Logger[F].debug(s"Starting to process chunk of size ${chunk.size}") + _ <- env.metrics.rawCount(chunk.size) + begin <- Sync[F].realTime + enrichments <- env.enrichments.get + result <- chunk.parTraverseN(env.streamsSettings.concurrency.enrich) { payload => + enrichWith[F, A]( + enrichments.registry, + env.adapterRegistry, + env.igluClient, + env.sentry, + env.processor, + env.featureFlags, + env.metrics.invalidCount, + env.registryLookup, + env.atomicFields, + env.sinkIncomplete.isDefined, + env.maxJsonDepth, + env.getPayload, + payload + ) + } + end <- Sync[F].realTime + _ <- Logger[F].debug(s"Chunk of size ${chunk.size} enriched in ${end - begin} ms") + } yield result.toList + } + + val enrichedStream: Stream[F, List[Result[A]]] = env.source.chunks.evalMap(enrichChunk) + + val sinkAndCheckpoint: Pipe[F, List[Result[A]], Unit] = + _.parEvalMap(env.streamsSettings.concurrency.sink) { chunk => for { - begin <- Clock[F].realTime - result <- sinkChunk(chunk.map(_._2), env).as(chunk.map(_._1)) - end <- Clock[F].realTime + begin <- Sync[F].realTime + _ <- sinkChunk(chunk, env) + end <- Sync[F].realTime _ <- Logger[F].debug(s"Chunk of size ${chunk.size} sunk in ${end - begin}") - } yield result - ) + } yield chunk.map(_.original) + } .evalMap(env.checkpoint) - enriched.through(CleanCancellation(sinkAndCheckpoint)) + enrichedStream.through(CleanCancellation(sinkAndCheckpoint)) } /** * Enrich a single `CollectorPayload` to get list of bad rows and/or enriched events * @return enriched event or bad row, along with the collector timestamp */ - def enrichWith[F[_]: Sync]( - enrichRegistry: F[EnrichmentRegistry[F]], + def enrichWith[F[_]: Sync, A]( + enrichRegistry: EnrichmentRegistry[F], adapterRegistry: AdapterRegistry[F], igluClient: IgluCirceClient[F], sentry: Option[SentryClient], @@ -121,20 +127,20 @@ object Enrich { invalidCount: F[Unit], registryLookup: RegistryLookup[F], atomicFields: AtomicFields, - emitIncomplete: Boolean - )( - row: Array[Byte] - ): F[Result] = { - val payload = ThriftLoader.toCollectorPayload(row, processor, featureFlags.tryBase64Decoding) - val collectorTstamp = payload.toOption.flatMap(_.flatMap(_.context.timestamp).map(_.getMillis)) - - val result = - for { - etlTstamp <- Clock[F].realTime.map(time => new DateTime(time.toMillis)) - registry <- enrichRegistry - enriched <- EtlPipeline.processEvents[F]( + emitIncomplete: Boolean, + maxJsonDepth: Int, + getPayload: A => Array[Byte], + row: A + ): F[Result[A]] = + for { + bytes <- Sync[F].delay(getPayload(row)) + payload <- Sync[F].delay(ThriftLoader.toCollectorPayload(bytes, processor, featureFlags.tryBase64Decoding)) + etlTstamp <- Sync[F].realTime.map(time => new DateTime(time.toMillis)) + collectorTstamp = payload.toOption.flatMap(_.flatMap(_.context.timestamp).map(_.getMillis)) + enriched <- EtlPipeline + .processEvents[F]( adapterRegistry, - registry, + enrichRegistry, igluClient, processor, etlTstamp, @@ -143,37 +149,35 @@ object Enrich { invalidCount, registryLookup, atomicFields, - emitIncomplete + emitIncomplete, + maxJsonDepth ) - } yield (enriched, collectorTstamp) - - result.handleErrorWith(sendToSentry[F](row, sentry, processor, collectorTstamp)) - } + .handleErrorWith(recoverFromError[F](bytes, sentry, processor)) + } yield Result(row, enriched, collectorTstamp) /** Stringify `ThriftLoader` result for debugging purposes */ def payloadToString(payload: ValidatedNel[BadRow.CPFormatViolation, Option[CollectorPayload]]): String = payload.fold(_.asJson.noSpaces, _.map(_.toBadRowPayload.asJson.noSpaces).getOrElse("None")) /** Log an error, turn the problematic `CollectorPayload` into `BadRow` and notify Sentry if configured */ - def sendToSentry[F[_]: Sync: Clock]( - original: Array[Byte], + def recoverFromError[F[_]: Sync]( + bytes: Array[Byte], sentry: Option[SentryClient], - processor: Processor, - collectorTstamp: Option[Long] + processor: Processor )( error: Throwable - ): F[Result] = + ): F[List[Enriched]] = for { _ <- Logger[F].error("Runtime exception during payload enrichment. CollectorPayload converted to generic_error and ack'ed") - now <- Clock[F].realTimeInstant - badRow = genericBadRow(original, now, error, processor) + now <- Sync[F].realTimeInstant + badRow = genericBadRow(bytes, now, error, processor) _ <- sentry match { case Some(client) => Sync[F].delay(client.sendException(error)) case None => Sync[F].unit } - } yield (List(Ior.left(badRow)), collectorTstamp) + } yield List(Ior.left(badRow)) /** Build a `generic_error` bad row for unhandled runtime errors */ def genericBadRow( @@ -189,12 +193,12 @@ object Enrich { } def sinkChunk[F[_]: Parallel: Sync, A]( - chunk: List[Result], + chunk: List[Result[A]], env: Environment[F, A] ): F[Unit] = { val (bad, enriched, incomplete) = chunk - .flatMap(_._1) + .flatMap(_.enriched) .foldLeft((List.empty[BadRow], List.empty[EnrichedEvent], List.empty[EnrichedEvent])) { case (previous, item) => val (bad, enriched, incomplete) = previous @@ -228,7 +232,7 @@ object Enrich { env.piiAttributes, env.processor, env.streamsSettings.maxRecordSize - ) *> env.metrics.enrichLatency(chunk.headOption.flatMap(_._2)), + ) *> env.metrics.enrichLatency(chunk.headOption.flatMap(_.collectorTstamp)), sinkBad(allBad, env.sinkBad, env.metrics.badCount), if (incompleteTooBig.nonEmpty) Logger[F].warn(s"${incompleteTooBig.size} incomplete events discarded because they are too big") else Sync[F].unit, diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Environment.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Environment.scala index 0858b1f64..971671be3 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Environment.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Environment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -129,7 +129,8 @@ final case class Environment[F[_], A]( region: Option[String], cloud: Option[Cloud], featureFlags: FeatureFlags, - atomicFields: AtomicFields + atomicFields: AtomicFields, + maxJsonDepth: Int ) object Environment { @@ -210,10 +211,10 @@ object Environment { incomplete <- sinkIncomplete.sequence http4s <- Clients.mkHttp() clts <- clients.map(Clients.init(http4s, _)) - igluClient <- IgluCirceClient.parseDefault[F](parsedConfigs.igluJson).resource + igluClient <- IgluCirceClient.parseDefault[F](parsedConfigs.igluJson, parsedConfigs.configFile.maxJsonDepth).resource remoteAdaptersEnabled = file.remoteAdapters.configs.nonEmpty metrics <- Resource.eval(Metrics.build[F](file.monitoring.metrics, remoteAdaptersEnabled, incomplete.isDefined)) - metadata <- Resource.eval(metadataReporter[F](file, processor.artifact, http4s)) + metadata <- metadataReporter[F](file, processor.artifact, http4s) assets = parsedConfigs.enrichmentConfigs.flatMap(_.filesToCache) remoteAdapters <- prepareRemoteAdapters[F](file.remoteAdapters, metrics) adapterRegistry = new AdapterRegistry(remoteAdapters, file.adaptersSchemas) @@ -252,7 +253,8 @@ object Environment { getRegionFromConfig(file).orElse(getRegion), cloud, featureFlags, - atomicFields + atomicFields, + parsedConfigs.configFile.maxJsonDepth ) } @@ -275,11 +277,16 @@ object Environment { config: ConfigFile, appName: String, httpClient: Http4sClient[F] - ): F[Metadata[F]] = - config.experimental - .flatMap(_.metadata) - .map(metadataConfig => Metadata.build[F](metadataConfig, Metadata.HttpMetadataReporter[F](metadataConfig, appName, httpClient))) - .getOrElse(Metadata.noop[F].pure[F]) + ): Resource[F, Metadata[F]] = + config.experimental.flatMap(_.metadata) match { + case Some(metadataConfig) => + for { + reporter <- Metadata.HttpMetadataReporter.resource(metadataConfig, appName, httpClient) + metadata <- Resource.eval(Metadata.build(metadataConfig, reporter)) + } yield metadata + case None => + Resource.pure(Metadata.noop[F]) + } private implicit class EitherTOps[F[_], E: Show, A](eitherT: EitherT[F, E, A]) { def resource(implicit F: Sync[F]): Resource[F, A] = { diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Run.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Run.scala index af246637d..f4168e7f0 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Run.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Run.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Telemetry.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Telemetry.scala index dede324fe..0777bef78 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Telemetry.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/Telemetry.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Base64Hocon.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Base64Hocon.scala index b9e833572..cd2248b48 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Base64Hocon.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Base64Hocon.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/CliConfig.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/CliConfig.scala index 0e9d67b2d..50781d341 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/CliConfig.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/CliConfig.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ConfigFile.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ConfigFile.scala index 6ffb12f61..21b7ce03f 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ConfigFile.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ConfigFile.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -51,7 +51,8 @@ final case class ConfigFile( adaptersSchemas: AdaptersSchemas, blobStorage: BlobStorageClients, license: License, - validation: Validation + validation: Validation, + maxJsonDepth: Int ) object ConfigFile { diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ParsedConfigs.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ParsedConfigs.scala index f1bd47a25..858fd5b08 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ParsedConfigs.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ParsedConfigs.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -85,7 +85,7 @@ object ParsedConfigs { resolverConfig <- EitherT.fromEither[F](Resolver.parseConfig(igluJson)).leftMap(x => show"Cannot decode Iglu resolver from provided json. $x") resolver <- Resolver.fromConfig[F](resolverConfig).leftMap(x => show"Cannot create Iglu resolver from provided json. $x") - client <- EitherT.liftF(IgluCirceClient.fromResolver[F](resolver, resolverConfig.cacheSize)) + client <- EitherT.liftF(IgluCirceClient.fromResolver[F](resolver, resolverConfig.cacheSize, configFile.maxJsonDepth)) _ <- EitherT.liftF( Logger[F].info(show"Parsed Iglu Client with following registries: ${resolver.repos.map(_.config.name).mkString(", ")}") ) diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Sentry.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Sentry.scala index 50e1695b2..e223d5d98 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Sentry.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Sentry.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/io.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/io.scala index 633781548..f37649356 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/io.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/io.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/package.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/package.scala index ae4405f5a..a00f43fce 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/package.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/package.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Clients.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Clients.scala index 215456ce0..40664c456 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Clients.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Clients.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSink.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSink.scala index 86d05eba8..7c1b082bb 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSink.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSystem.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSystem.scala index 0d72eb666..d00b6ce8d 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSystem.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/FileSystem.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Metrics.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Metrics.scala index 7bfe9c212..3be08153c 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Metrics.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Metrics.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Retries.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Retries.scala index 6afc06878..407130f19 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Retries.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Retries.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Source.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Source.scala index 4957f7c6c..9f27b3031 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Source.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/Source.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/StatsDReporter.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/StatsDReporter.scala index 9a876d531..f37dad876 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/StatsDReporter.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/StatsDReporter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/experimental/Metadata.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/experimental/Metadata.scala index 3f3360246..4bd907689 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/experimental/Metadata.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/experimental/Metadata.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -79,7 +79,7 @@ object Metadata { new Metadata[F] { def report: Stream[F, Unit] = for { - _ <- Stream.eval(Logger[F].info("Starting metadata repoter")) + _ <- Stream.eval(Logger[F].info("Starting metadata reporter")) _ <- Stream.bracket(Sync[F].unit)(_ => submit(reporter, observedRef)) _ <- Stream.fixedDelay[F](config.interval) _ <- Stream.eval(submit(reporter, observedRef)) @@ -103,6 +103,7 @@ object Metadata { case (event, entitiesAndCount) => reporter.report(snapshot.periodStart, snapshot.periodEnd, event, entitiesAndCount) } + _ <- reporter.flush() } yield () trait MetadataReporter[F[_]] { @@ -112,14 +113,39 @@ object Metadata { event: MetadataEvent, entitiesAndCount: EntitiesAndCount ): F[Unit] + + def flush(): F[Unit] } - case class HttpMetadataReporter[F[_]: Async]( + case class HttpMetadataReporter[F[_]: Sync]( config: MetadataConfig, - appName: String, - client: Client[F] + tracker: Tracker[F] ) extends MetadataReporter[F] { - def initTracker( + + def report( + periodStart: Instant, + periodEnd: Instant, + event: MetadataEvent, + entitiesAndCount: EntitiesAndCount + ): F[Unit] = + Logger[F].debug(s"Tracking observed event ${event.schema.toSchemaUri}") >> + tracker.trackSelfDescribingEvent( + mkWebhookEvent(config.organizationId, config.pipelineId, periodStart, periodEnd, event, entitiesAndCount.count), + mkWebhookContexts(entitiesAndCount.entities).toSeq + ) + + def flush(): F[Unit] = tracker.flushEmitters() + } + + object HttpMetadataReporter { + def resource[F[_]: Async]( + config: MetadataConfig, + appName: String, + client: Client[F] + ): Resource[F, HttpMetadataReporter[F]] = + initTracker(config, appName, client).map(t => HttpMetadataReporter(config, t)) + + private def initTracker[F[_]: Async]( config: MetadataConfig, appName: String, client: Client[F] @@ -133,33 +159,20 @@ object Metadata { https = config.endpoint.scheme.map(_ == Uri.Scheme.https).getOrElse(false) ), client, + bufferConfig = Emitter.BufferConfig.PayloadSize(100000), retryPolicy = Emitter.RetryPolicy.MaxAttempts(10), - callback = Some(emitterCallback _) + callback = Some(emitterCallback[F](_, _, _)) ) } yield new Tracker(NonEmptyList.of(emitter), "tracker-metadata", appName) - def report( - periodStart: Instant, - periodEnd: Instant, - event: MetadataEvent, - entitiesAndCount: EntitiesAndCount - ): F[Unit] = - initTracker(config, appName, client).use { t => - Logger[F].info(s"Tracking observed event ${event.schema.toSchemaUri}") >> - t.trackSelfDescribingEvent( - mkWebhookEvent(config.organizationId, config.pipelineId, periodStart, periodEnd, event, entitiesAndCount.count), - mkWebhookContexts(entitiesAndCount.entities).toSeq - ) >> t.flushEmitters() - } - - private def emitterCallback( + private def emitterCallback[F[_]: Sync]( params: Emitter.EndpointParams, req: Emitter.Request, res: Emitter.Result ): F[Unit] = res match { case Emitter.Result.Success(_) => - Logger[F].info(s"Metadata successfully sent to ${params.getGetUri}") + Logger[F].debug(s"Metadata successfully sent to ${params.getUri}") case Emitter.Result.Failure(code) => Logger[F].warn(s"Sending metadata got unexpected HTTP code $code from ${params.getUri}") case Emitter.Result.TrackerFailure(exception) => diff --git a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/package.scala b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/package.scala index 40cd78efe..d30442cd7 100644 --- a/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/package.scala +++ b/modules/common-fs2/src/main/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/package.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -26,8 +26,4 @@ package object fs2 { type AttributedByteSink[F[_]] = List[AttributedData[Array[Byte]]] => F[Unit] type Enriched = Ior[BadRow, EnrichedEvent] - type Result = (List[Enriched], Option[Long]) - - /** Function to transform an origin raw payload into good and/or bad rows */ - type Enrich[F[_]] = Array[Byte] => F[Result] } diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/AssetsSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/AssetsSpec.scala index 6f4efcce0..ef96a120e 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/AssetsSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/AssetsSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/EnrichSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/EnrichSpec.scala index 31c206697..5c4b31de2 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/EnrichSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/EnrichSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -42,7 +42,7 @@ import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.{IpLook import com.snowplowanalytics.snowplow.enrich.common.enrichments.{AtomicFields, MiscEnrichments} import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent -import com.snowplowanalytics.snowplow.enrich.common.utils.ConversionUtils +import com.snowplowanalytics.snowplow.enrich.common.utils.{ConversionUtils, IgluUtilsSpec, JsonUtilsSpec} import com.snowplowanalytics.snowplow.enrich.common.fs2.config.io.FeatureFlags import com.snowplowanalytics.snowplow.enrich.common.fs2.EnrichSpec.{Expected, minimalEvent, normalizeResult} import com.snowplowanalytics.snowplow.enrich.common.SpecHelpers @@ -50,6 +50,8 @@ import com.snowplowanalytics.snowplow.enrich.common.fs2.test._ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { + override protected val Timeout = 1.minutes + sequential "enrichWith" should { @@ -69,7 +71,7 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { SpecHelpers.createIgluClient(List(TestEnvironment.embeddedRegistry)).flatMap { igluClient => Enrich .enrichWith( - TestEnvironment.enrichmentReg.pure[IO], + TestEnvironment.enrichmentReg, TestEnvironment.adapterRegistry, igluClient, None, @@ -78,8 +80,9 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { IO.unit, SpecHelpers.registryLookup, AtomicFields.from(valueLimits = Map.empty), - SpecHelpers.emitIncomplete - )( + SpecHelpers.emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth, + identity[Array[Byte]], EnrichSpec.payload ) .map(normalizeResult) @@ -100,7 +103,7 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { .flatMap { igluClient => Enrich .enrichWith( - TestEnvironment.enrichmentReg.pure[IO], + TestEnvironment.enrichmentReg, TestEnvironment.adapterRegistry, igluClient, None, @@ -109,8 +112,9 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { IO.unit, SpecHelpers.registryLookup, AtomicFields.from(valueLimits = Map.empty), - SpecHelpers.emitIncomplete - )( + SpecHelpers.emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth, + identity[Array[Byte]], payload ) .map(normalizeResult) @@ -139,7 +143,7 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { SpecHelpers.createIgluClient(List(TestEnvironment.embeddedRegistry)).flatMap { igluClient => Enrich .enrichWith( - TestEnvironment.enrichmentReg.pure[IO], + TestEnvironment.enrichmentReg, TestEnvironment.adapterRegistry, igluClient, None, @@ -148,8 +152,9 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { IO.unit, SpecHelpers.registryLookup, AtomicFields.from(valueLimits = Map.empty), - SpecHelpers.emitIncomplete - )( + SpecHelpers.emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth, + identity[Array[Byte]], Base64.getEncoder.encode(EnrichSpec.payload) ) .map(normalizeResult) @@ -164,7 +169,7 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { SpecHelpers.createIgluClient(List(TestEnvironment.embeddedRegistry)).flatMap { igluClient => Enrich .enrichWith( - TestEnvironment.enrichmentReg.pure[IO], + TestEnvironment.enrichmentReg, TestEnvironment.adapterRegistry, igluClient, None, @@ -173,8 +178,9 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { IO.unit, SpecHelpers.registryLookup, AtomicFields.from(valueLimits = Map.empty), - SpecHelpers.emitIncomplete - )( + SpecHelpers.emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth, + identity[Array[Byte]], Base64.getEncoder.encode(EnrichSpec.payload) ) .map(normalizeResult) @@ -291,6 +297,90 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { } } } + + "send events with deeply nested JSON entities to bad stream" in { + def payloadDeepContext(depth: Int) = { + val deepJsonObject = JsonUtilsSpec.createDeepJsonObject(depth) + EnrichSpec.collectorPayload.copy( + querystring = EnrichSpec.jsonEntityQueryParam( + "cx", + IgluUtilsSpec.buildInputContexts(List(deepJsonObject)) + ) :: EnrichSpec.querystring + ) + } + + def payloadDeepUnstruct(depth: Int) = { + val deepJsonArray = JsonUtilsSpec.createDeepJsonArray(depth) + EnrichSpec.collectorPayload.copy( + querystring = EnrichSpec.jsonEntityQueryParam( + "ue_px", + IgluUtilsSpec.buildUnstruct(deepJsonArray) + ) :: EnrichSpec.querystring + ) + } + + val input = Stream.emits( + List( + EnrichSpec.payload, + payloadDeepContext(10000).toRaw, + payloadDeepUnstruct(10000).toRaw, + payloadDeepContext(1000000).toRaw, + payloadDeepUnstruct(1000000).toRaw + ) + ) + + def badRowCheck(bad: Vector[BadRow]) = { + val fieldsSchemaViolation = bad.collect { + case BadRow.SchemaViolations( + _, + Failure.SchemaViolations( + _, + NonEmptyList( + FailureDetails.SchemaViolation.NotJson( + field, + _, + "invalid json: maximum allowed JSON depth exceeded" + ), + Nil + ) + ), + _ + ) => + field + } + val sizeViolationBadRows = bad.filter { + case _: BadRow.SizeViolation => true + case _ => false + } + (fieldsSchemaViolation must contain(exactly("contexts", "unstruct"))) + (sizeViolationBadRows.size must_== 2) + (bad.size must_== 4) + } + + def incompleteCheck(incomplete: Vector[Event]) = { + incomplete must contain( + beLike[Event] { + case event + if event.contexts.data.size.isEmpty && + event.unstruct_event.data.isEmpty => + ok + } + ).forall + (incomplete.size must_== 2) + } + + TestEnvironment.make(input).use { test => + test + .run(streamTimeout = 10.seconds) + .map { + case (bad, pii, good, incomplete) => + badRowCheck(bad) + incompleteCheck(incomplete) + (pii must be empty) + (good must contain(exactly(Expected))) + } + } + } } "sinkChunk" should { @@ -498,7 +588,7 @@ class EnrichSpec extends Specification with CatsEffect with ScalaCheck { def sinkOne( environment: Environment[IO, Array[Byte]], event: Ior[BadRow, EnrichedEvent] - ): IO[Unit] = Enrich.sinkChunk(List((List(event), None)), environment) + ): IO[Unit] = Enrich.sinkChunk(List(Enrich.Result(Array.emptyByteArray, List(event), None)), environment) } object EnrichSpec { @@ -530,8 +620,8 @@ object EnrichSpec { Validated.Invalid(badRow) } - def normalizeResult(payload: Result): List[Ior[BadRow, Event]] = - payload._1.map { + def normalizeResult(payload: Enrich.Result[Any]): List[Ior[BadRow, Event]] = + payload.enriched.map { case Ior.Right(enriched) => normalize(ConversionUtils.tabSeparatedEnrichedEvent(enriched)).toIor case Ior.Left(err) => Ior.Left(err) case Ior.Both(_, enriched) => normalize(ConversionUtils.tabSeparatedEnrichedEvent(enriched)).toIor @@ -559,4 +649,14 @@ object EnrichSpec { ) val featureFlags = FeatureFlags(acceptInvalid = false, legacyEnrichmentOrder = false, tryBase64Decoding = false) + + def jsonEntityQueryParam(fieldName: String, jsonEntity: String): BasicNameValuePair = + new BasicNameValuePair( + fieldName, + new String( + Base64.getEncoder.encode( + jsonEntity.getBytes() + ) + ) + ) } diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/EventGenEtlPipelineSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/EventGenEtlPipelineSpec.scala index 55fee9718..fb42e2991 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/EventGenEtlPipelineSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/EventGenEtlPipelineSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -166,7 +166,8 @@ class EventGenEtlPipelineSpec extends Specification with CatsEffect { val adapterRegistry = new AdapterRegistry(Map.empty[(String, String), RemoteAdapter[IO]], SpecHelpers.adaptersSchemas) val enrichmentReg = EnrichmentRegistry[IO]() val igluCentral = Registry.IgluCentral - val client = IgluCirceClient.parseDefault[IO](json""" + val client = IgluCirceClient.parseDefault[IO]( + json""" { "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-1", "data": { @@ -195,7 +196,9 @@ class EventGenEtlPipelineSpec extends Specification with CatsEffect { ] } } - """) + """, + maxJsonDepth = 40 + ) val processor = Processor("sce-test-suite", "1.0.0") val dateTime = DateTime.now() val process = Processor("EventGenEtlPipelineSpec", "v1") @@ -212,7 +215,8 @@ class EventGenEtlPipelineSpec extends Specification with CatsEffect { IO.unit, SpecHelpers.registryLookup, AtomicFields.from(Map.empty), - SpecHelpers.emitIncomplete + SpecHelpers.emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) def rethrowBadRows[A]: Pipe[IO, ValidatedNel[BadRow, A], A] = diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/PayloadGen.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/PayloadGen.scala index 65f1632e1..4656d269c 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/PayloadGen.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/PayloadGen.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/BlackBoxTesting.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/BlackBoxTesting.scala index ee658f36d..00a6a9086 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/BlackBoxTesting.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/BlackBoxTesting.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -96,7 +96,7 @@ object BlackBoxTesting extends Specification with CatsEffect { getEnrichmentRegistry(enrichmentConfig, igluClient).use { registry => Enrich .enrichWith( - IO.pure(registry), + registry, TestEnvironment.adapterRegistry, igluClient, None, @@ -105,12 +105,14 @@ object BlackBoxTesting extends Specification with CatsEffect { IO.unit, SpecHelpers.registryLookup, AtomicFields.from(valueLimits = Map.empty), - SpecHelpers.emitIncomplete - )( + SpecHelpers.emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth, + identity[Array[Byte]], input ) + .map(_.enriched) .map { - case (List(Ior.Right(enriched)), _) => checkEnriched(enriched, expected) + case List(Ior.Right(enriched)) => checkEnriched(enriched, expected) case other => ko(s"there should be one enriched event but got $other") } } diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/CallrailAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/CallrailAdapterSpec.scala index 3e7ccc463..b102b32dd 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/CallrailAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/CallrailAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/GoogleAnalyticsAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/GoogleAnalyticsAdapterSpec.scala index c11e1e4dd..f52e75c23 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/GoogleAnalyticsAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/GoogleAnalyticsAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/HubspotAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/HubspotAdapterSpec.scala index eaa7a1955..d8c40eddf 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/HubspotAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/HubspotAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MailchimpAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MailchimpAdapterSpec.scala index be36112b5..cb5fde4a9 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MailchimpAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MailchimpAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MailgunAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MailgunAdapterSpec.scala index e29fdf287..c01c572e8 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MailgunAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MailgunAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MandrillAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MandrillAdapterSpec.scala index 84be5ff16..d5bab9369 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MandrillAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MandrillAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MarketoAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MarketoAdapterSpec.scala index 7ea29708c..7a9919d07 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MarketoAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/MarketoAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/OlarkAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/OlarkAdapterSpec.scala index 456de3572..45792cef9 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/OlarkAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/OlarkAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/PagerDutyAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/PagerDutyAdapterSpec.scala index d0dfc7406..d1c94f690 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/PagerDutyAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/PagerDutyAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/PingdomAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/PingdomAdapterSpec.scala index 806609910..00c48d178 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/PingdomAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/PingdomAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/SendgridAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/SendgridAdapterSpec.scala index b8363bf38..5591f2453 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/SendgridAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/SendgridAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/StatusGatorAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/StatusGatorAdapterSpec.scala index 138f7054f..f777abac2 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/StatusGatorAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/StatusGatorAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/Tp2AdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/Tp2AdapterSpec.scala index 0aa68adab..001171603 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/Tp2AdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/Tp2AdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -35,7 +35,7 @@ class Tp2AdapterSpec extends Specification with CatsEffect { SpecHelpers.createIgluClient(List(TestEnvironment.embeddedRegistry)).flatMap { igluClient => Enrich .enrichWith( - TestEnvironment.enrichmentReg.pure[IO], + TestEnvironment.enrichmentReg, TestEnvironment.adapterRegistry, igluClient, None, @@ -44,12 +44,14 @@ class Tp2AdapterSpec extends Specification with CatsEffect { IO.unit, SpecHelpers.registryLookup, AtomicFields.from(valueLimits = Map.empty), - SpecHelpers.emitIncomplete - )( + SpecHelpers.emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth, + identity[Array[Byte]], input ) + .map(_.enriched) .map { - case (l, _) if l.forall(_.isRight) => l must haveSize(10) + case l if l.forall(_.isRight) => l must haveSize(10) case other => ko(s"there should be 10 enriched events, got $other") } } diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/UnbounceAdapterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/UnbounceAdapterSpec.scala index ae50b436d..1520c4c84 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/UnbounceAdapterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/adapters/UnbounceAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/ApiRequestEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/ApiRequestEnrichmentSpec.scala index b67b38b50..a6cf5e8d4 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/ApiRequestEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/ApiRequestEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CampaignAttributionEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CampaignAttributionEnrichmentSpec.scala index 0fa7fdf2d..0a733b283 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CampaignAttributionEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CampaignAttributionEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CurrencyConversionEnrichmentTransactionItemSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CurrencyConversionEnrichmentTransactionItemSpec.scala index 43aa90e8f..7996218db 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CurrencyConversionEnrichmentTransactionItemSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CurrencyConversionEnrichmentTransactionItemSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CurrencyConversionEnrichmentTransactionSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CurrencyConversionEnrichmentTransactionSpec.scala index 90a2981ea..4aaabfbea 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CurrencyConversionEnrichmentTransactionSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/CurrencyConversionEnrichmentTransactionSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/IabEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/IabEnrichmentSpec.scala index 89aed1b01..34e8d7a4c 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/IabEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/IabEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/IpLookupsEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/IpLookupsEnrichmentSpec.scala index 34c848286..0a3fcfab4 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/IpLookupsEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/IpLookupsEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/PiiEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/PiiEnrichmentSpec.scala index 2455c339e..6888d062d 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/PiiEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/PiiEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/RefererParserEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/RefererParserEnrichmentSpec.scala index 9d09757a3..70b253a0c 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/RefererParserEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/RefererParserEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/SqlQueryEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/SqlQueryEnrichmentSpec.scala index c3056e6c5..dd5d860c8 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/SqlQueryEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/SqlQueryEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala index 89a4f4e18..5cb7a4c55 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/enrichments/YauaaEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/StructEventSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/StructEventSpec.scala index 5dc518dcd..b3b041cb6 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/StructEventSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/StructEventSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/TransactionItemSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/TransactionItemSpec.scala index 17c081c38..c65cc9695 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/TransactionItemSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/TransactionItemSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/TransactionSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/TransactionSpec.scala index b7ca07f6c..391f318cd 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/TransactionSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/TransactionSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/UnstructEventB64Spec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/UnstructEventB64Spec.scala index 1981bb6a5..9cb9e3e27 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/UnstructEventB64Spec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/UnstructEventB64Spec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/UnstructEventSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/UnstructEventSpec.scala index 22fe8bcac..15a5b1bae 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/UnstructEventSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/blackbox/misc/UnstructEventSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Base64HoconSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Base64HoconSpec.scala index 860151d90..2622e694f 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Base64HoconSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/Base64HoconSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/CliConfigSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/CliConfigSpec.scala index 6f27827b5..26bd52410 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/CliConfigSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/CliConfigSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ConfigFileSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ConfigFileSpec.scala index f32300843..1ebe31cbe 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ConfigFileSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ConfigFileSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ParsedConfigsSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ParsedConfigsSpec.scala index effde6e90..e7fa00307 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ParsedConfigsSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/config/ParsedConfigsSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -123,7 +123,8 @@ class ParsedConfigsSpec extends Specification with CatsEffect { adaptersSchemas, BlobStorageClients(gcs = false, s3 = true, azureStorage = None), License(accept = true), - Validation(AtomicFields.from(valueLimits = Map.empty)) + Validation(AtomicFields.from(valueLimits = Map.empty)), + maxJsonDepth = 40 ) ParsedConfigs.validateConfig[IO](configFile).value.map(result => result must beLeft) diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/ApiRequestEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/ApiRequestEnrichmentSpec.scala index fc7f003ad..b2e3155cc 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/ApiRequestEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/ApiRequestEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/IabEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/IabEnrichmentSpec.scala index 504db244c..dc8e79f8d 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/IabEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/IabEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala index 254bc6201..b1a840acc 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/enrichments/YauaaEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/MetadataSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/MetadataSpec.scala index 9e1d11880..60afa0270 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/MetadataSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/MetadataSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -16,6 +16,7 @@ import java.time.Instant import scala.concurrent.duration._ import cats.effect.IO +import cats.Applicative import cats.effect.kernel.Ref import cats.effect.testing.specs2.CatsEffect import org.http4s.Uri @@ -34,7 +35,7 @@ class MetadataSpec extends Specification with CatsEffect { event: MetadataEvent, entitiesAndCount: EntitiesAndCount ) - case class TestReporter[F[_]](state: Ref[F, List[Report]]) extends MetadataReporter[F] { + case class TestReporter[F[_]: Applicative](state: Ref[F, List[Report]]) extends MetadataReporter[F] { def report( periodStart: Instant, @@ -45,6 +46,8 @@ class MetadataSpec extends Specification with CatsEffect { state.update( _ :+ Report(periodStart, periodEnd, event, entitiesAndCount) ) + + def flush(): F[Unit] = Applicative[F].unit } "Metadata" should { diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/SinkSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/SinkSpec.scala index 3bf24ba01..4a9a62da1 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/SinkSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/SinkSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/StatsDReporterSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/StatsDReporterSpec.scala index bd275f2e8..9e3e2a7e9 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/StatsDReporterSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/io/StatsDReporterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/AggregatesSpec.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/AggregatesSpec.scala index 9f4245220..d6957a7d7 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/AggregatesSpec.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/AggregatesSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/Counter.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/Counter.scala index 1d8bf01fb..6e3481548 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/Counter.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/Counter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/HttpServer.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/HttpServer.scala index a3b1afb18..5bce602c2 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/HttpServer.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/HttpServer.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/TestEnvironment.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/TestEnvironment.scala index cf089ce93..80b932942 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/TestEnvironment.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/TestEnvironment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -70,7 +70,8 @@ case class TestEnvironment[A]( * [[TestEnvironment.make]] */ def run( - updateEnv: Environment[IO, A] => Environment[IO, A] = identity + updateEnv: Environment[IO, A] => Environment[IO, A] = identity, + streamTimeout: FiniteDuration = 5.seconds ): IO[(Vector[BadRow], Vector[Event], Vector[Event], Vector[Event])] = { val updatedEnv = updateEnv(env) val stream = Enrich @@ -85,7 +86,7 @@ case class TestEnvironment[A]( ) ) for { - _ <- stream.haltAfter(5.seconds).compile.drain + _ <- stream.haltAfter(streamTimeout).compile.drain goodVec <- good piiVec <- pii badVec <- bad @@ -162,7 +163,8 @@ object TestEnvironment extends CatsEffect { None, None, EnrichSpec.featureFlags, - AtomicFields.from(valueLimits = Map.empty) + AtomicFields.from(valueLimits = Map.empty), + SpecHelpers.DefaultMaxJsonDepth ) _ <- Resource.eval(logger.info("TestEnvironment initialized")) } yield TestEnvironment(environment, counter, goodRef.get, piiRef.get, badRef.get, incompleteRef.get) diff --git a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/package.scala b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/package.scala index 119a90803..84b944cd6 100644 --- a/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/package.scala +++ b/modules/common-fs2/src/test/scala/com/snowplowanalytics/snowplow/enrich/common/fs2/test/package.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/resources/reference.conf b/modules/common/src/main/resources/reference.conf index 226da5df1..3c1182a9a 100644 --- a/modules/common/src/main/resources/reference.conf +++ b/modules/common/src/main/resources/reference.conf @@ -148,5 +148,94 @@ "created": "iglu:com.getvero/created/jsonschema/1-0-0", "updated": "iglu:com.getvero/updated/jsonschema/1-0-0" } + }, + "maxJsonDepth": 40, + "validation": { + "atomicFieldsLimits": { + "app_id": 255, + "platform": 255, + "event": 128, + "event_id": 36, + "name_tracker": 128, + "v_tracker": 100, + "v_collector": 100, + "v_etl": 100, + "user_id": 255, + "user_ipaddress": 128, + "user_fingerprint": 128, + "domain_userid": 128, + "network_userid": 128, + "geo_country": 2, + "geo_region": 3, + "geo_city": 75, + "geo_zipcode": 15, + "geo_region_name": 100, + "ip_isp": 100, + "ip_organization": 128, + "ip_domain": 128, + "ip_netspeed": 100, + "page_url": 10000, # Different than the one in atomic schema (4096) + "page_title": 2000, + "page_referrer": 10000, # Different than the one in atomic schema (4096) + "page_urlscheme": 16, + "page_urlhost": 255, + "page_urlpath": 3000, + "page_urlquery": 6000, + "page_urlfragment": 3000, + "refr_urlscheme": 16, + "refr_urlhost": 255, + "refr_urlpath": 6000, + "refr_urlquery": 6000, + "refr_urlfragment": 3000, + "refr_medium": 25, + "refr_source": 50, + "refr_term": 255, + "mkt_clickid": 1000, # Different than the one in atomic schema (128) + "mkt_network": 64, + "mkt_medium": 255, + "mkt_source": 255, + "mkt_term": 255, + "mkt_content": 500, + "mkt_campaign": 255, + "se_category": 1000, + "se_action": 1000, + "se_label": 4096, + "se_property": 1000, + "tr_orderid": 255, + "tr_affiliation": 255, + "tr_city": 255, + "tr_state": 255, + "tr_country": 255, + "ti_orderid": 255, + "ti_sku": 255, + "ti_name": 255, + "ti_category": 255, + "useragent": 1000, + "br_name": 50, + "br_family": 50, + "br_version": 50, + "br_type": 50, + "br_renderengine": 50, + "br_lang": 255, + "br_colordepth": 12, + "os_name": 50, + "os_family": 50, + "os_manufacturer": 50, + "os_timezone": 255, + "dvce_type": 50, + "doc_charset": 128, + "tr_currency": 3, + "ti_currency": 3, + "base_currency": 3, + "geo_timezone": 64, + "etl_tags": 500, + "refr_domain_userid": 128, + "domain_sessionid": 128, + "event_vendor": 1000, + "event_name": 1000, + "event_format": 128, + "event_version": 128, + "event_fingerprint": 128 + } } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlPipeline.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlPipeline.scala index 6ac39c633..44f8bd635 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlPipeline.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/EtlPipeline.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -60,12 +60,13 @@ object EtlPipeline { invalidCount: F[Unit], registryLookup: RegistryLookup[F], atomicFields: AtomicFields, - emitIncomplete: Boolean + emitIncomplete: Boolean, + maxJsonDepth: Int ): F[List[Ior[BadRow, EnrichedEvent]]] = input match { case Validated.Valid(Some(payload)) => adapterRegistry - .toRawEvents(payload, client, processor, registryLookup) + .toRawEvents(payload, client, processor, registryLookup, maxJsonDepth) .flatMap { case Validated.Valid(rawEvents) => rawEvents.toList.traverse { event => @@ -80,7 +81,8 @@ object EtlPipeline { invalidCount, registryLookup, atomicFields, - emitIncomplete + emitIncomplete, + maxJsonDepth ) .value } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala index 668d1d0ac..375b1c591 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdapterRegistry.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -92,15 +92,16 @@ class AdapterRegistry[F[_]: Clock: Monad]( payload: CollectorPayload, client: IgluCirceClient[F], processor: Processor, - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Validated[BadRow, NonEmptyList[RawEvent]]] = (adapters.get((payload.api.vendor, payload.api.version)) match { case Some(adapter) => - adapter.toRawEvents(payload, client, registryLookup) + adapter.toRawEvents(payload, client, registryLookup, maxJsonDepth) case None => remoteAdapters.get((payload.api.vendor, payload.api.version)) match { case Some(adapter) => - adapter.toRawEvents(payload) + adapter.toRawEvents(payload, maxJsonDepth) case None => val f = FailureDetails.AdapterFailure.InputData( "vendor/version", diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdaptersSchemas.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdaptersSchemas.scala index 3e76d2695..45a2ec8db 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdaptersSchemas.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/AdaptersSchemas.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/RawEvent.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/RawEvent.scala index 5047d4482..69a7f717f 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/RawEvent.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/RawEvent.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala index c9f551ebb..38756da00 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/Adapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -117,7 +117,8 @@ trait Adapter { def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] /** diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CallrailAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CallrailAdapter.scala index 40b233a4a..10af11c7a 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CallrailAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CallrailAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -66,7 +66,8 @@ case class CallrailAdapter(schemas: CallrailSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = { val _ = client val params = toMap(payload.querystring) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CloudfrontAccessLogAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CloudfrontAccessLogAdapter.scala index 5eb5f232c..f03745230 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CloudfrontAccessLogAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/CloudfrontAccessLogAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -82,7 +82,8 @@ case class CloudfrontAccessLogAdapter(schemas: CloudfrontAccessLogSchemas) exten override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = payload.body match { case Some(p) => diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala index ecd0fa344..5a9d73c72 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/GoogleAnalyticsAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -477,7 +477,8 @@ case class GoogleAnalyticsAdapter(schemas: GoogleAnalyticsSchemas) extends Adapt override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = { val events: Option[NonEmptyList[ValidatedNel[FailureDetails.AdapterFailure, RawEvent]]] = for { body <- payload.body diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/HubSpotAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/HubSpotAdapter.scala index f6d32c164..6ae40e274 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/HubSpotAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/HubSpotAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -67,7 +67,8 @@ case class HubSpotAdapter(schemas: HubspotSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -90,7 +91,7 @@ case class HubSpotAdapter(schemas: HubspotSchemas) extends Adapter { ) Monad[F].pure(failure.invalidNel) case (Some(body), _) => - payloadBodyToEvents(body) match { + payloadBodyToEvents(body, maxJsonDepth) match { case Left(f) => Monad[F].pure(f.invalidNel) case Right(list) => val _ = client @@ -130,10 +131,10 @@ case class HubSpotAdapter(schemas: HubspotSchemas) extends Adapter { * @param body The payload body from the HubSpot event * @return either a Successful List of JValue JSONs or a Failure String */ - private[registry] def payloadBodyToEvents(body: String): Either[FailureDetails.AdapterFailure, List[Json]] = + private[registry] def payloadBodyToEvents(body: String, maxJsonDepth: Integer): Either[FailureDetails.AdapterFailure, List[Json]] = for { b <- JsonUtils - .extractJson(body) + .extractJson(body, maxJsonDepth) .leftMap(e => FailureDetails.AdapterFailure.NotJson("body", body.some, e)) a <- b.asArray.toRight( FailureDetails.AdapterFailure.InputData("body", body.some, "not a json array") diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala index b2f49ce6d..17e227697 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/IgluAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -66,7 +66,8 @@ object IgluAdapter extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = { val _ = client val params = toMap(payload.querystring) @@ -77,9 +78,9 @@ object IgluAdapter extends Adapter { FailureDetails.AdapterFailure.InputData("contentType", none, msg).invalidNel ) case (None, Some(body), Some(contentType)) => - Monad[F].pure(payloadSdJsonToEvent(payload, body, contentType, params)) + Monad[F].pure(payloadSdJsonToEvent(payload, body, contentType, params, maxJsonDepth)) case (Some(schemaUri), _, _) => // Ignore body - Monad[F].pure(payloadToEventWithSchema(payload, schemaUri, params)) + Monad[F].pure(payloadToEventWithSchema(payload, schemaUri, params, maxJsonDepth)) case (None, None, _) => val nel = NonEmptyList.of( FailureDetails.AdapterFailure @@ -103,11 +104,12 @@ object IgluAdapter extends Adapter { payload: CollectorPayload, body: String, contentType: String, - params: Map[String, Option[String]] + params: Map[String, Option[String]], + maxJsonDepth: Int ): ValidatedNel[FailureDetails.AdapterFailure, NonEmptyList[RawEvent]] = contentType match { - case contentTypes._1 => sdJsonBodyToEvent(payload, body, params) - case contentTypes._2 => sdJsonBodyToEvent(payload, body, params) + case contentTypes._1 => sdJsonBodyToEvent(payload, body, params, maxJsonDepth) + case contentTypes._2 => sdJsonBodyToEvent(payload, body, params, maxJsonDepth) case _ => val msg = s"expected one of ${List(contentTypes._1, contentTypes._2).mkString(", ")}" FailureDetails.AdapterFailure @@ -124,9 +126,10 @@ object IgluAdapter extends Adapter { private[registry] def sdJsonBodyToEvent( payload: CollectorPayload, body: String, - params: Map[String, Option[String]] + params: Map[String, Option[String]], + maxJsonDepth: Int ): ValidatedNel[FailureDetails.AdapterFailure, NonEmptyList[RawEvent]] = - JsonUtils.extractJson(body) match { + JsonUtils.extractJson(body, maxJsonDepth) match { case Right(parsed) => SelfDescribingData.parse(parsed) match { case Left(parseError) => @@ -165,7 +168,8 @@ object IgluAdapter extends Adapter { private[registry] def payloadToEventWithSchema( payload: CollectorPayload, schemaUri: String, - params: Map[String, Option[String]] + params: Map[String, Option[String]], + maxJsonDepth: Int ): ValidatedNel[FailureDetails.AdapterFailure, NonEmptyList[RawEvent]] = SchemaKey.fromUri(schemaUri) match { case Left(parseError) => @@ -194,8 +198,8 @@ object IgluAdapter extends Adapter { .valid case (Some(body), Some(contentType)) => contentType match { - case contentTypes._1 => jsonBodyToEvent(payload, body, key, params) - case contentTypes._2 => jsonBodyToEvent(payload, body, key, params) + case contentTypes._1 => jsonBodyToEvent(payload, body, key, params, maxJsonDepth) + case contentTypes._2 => jsonBodyToEvent(payload, body, key, params, maxJsonDepth) case contentTypes._3 => formBodyToEvent(payload, body, key, params) case _ => val msg = s"expected one of $contentTypesStr" @@ -223,7 +227,8 @@ object IgluAdapter extends Adapter { payload: CollectorPayload, body: String, schemaUri: SchemaKey, - params: Map[String, Option[String]] + params: Map[String, Option[String]], + maxJsonDepth: Int ): ValidatedNel[FailureDetails.AdapterFailure, NonEmptyList[RawEvent]] = { def buildRawEvent(e: Json): RawEvent = RawEvent( @@ -234,7 +239,7 @@ object IgluAdapter extends Adapter { context = payload.context ) - JsonUtils.extractJson(body) match { + JsonUtils.extractJson(body, maxJsonDepth) match { case Right(parsed) => parsed.asArray match { case Some(array) => diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailchimpAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailchimpAdapter.scala index 600405052..8e3cb75a3 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailchimpAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailchimpAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -72,7 +72,8 @@ case class MailchimpAdapter(schemas: MailchimpSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailgunAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailgunAdapter.scala index 7afb819b7..813e622a3 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailgunAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MailgunAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -17,7 +17,6 @@ import cats.syntax.either._ import cats.syntax.option._ import cats.syntax.validated._ import io.circe._ -import io.circe.parser._ import com.snowplowanalytics.iglu.client.IgluCirceClient import com.snowplowanalytics.iglu.client.resolver.registries.RegistryLookup @@ -61,7 +60,8 @@ case class MailgunAdapter(schemas: MailgunSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -93,9 +93,9 @@ case class MailgunAdapter(schemas: MailgunSchemas) extends Adapter { case (Some(body), Some(_)) => val _ = client val params = toMap(payload.querystring) - parse(body) match { + JU.extractJson(body, maxJsonDepth) match { case Left(e) => - val msg = s"could not parse body: ${JU.stripInstanceEtc(e.getMessage).orNull}" + val msg = s"could not parse body: ${JU.stripInstanceEtc(e).orNull}" Monad[F].pure( FailureDetails.AdapterFailure .InputData("body", body.some, msg) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MandrillAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MandrillAdapter.scala index 09979393a..c9079bbf6 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MandrillAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MandrillAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -67,7 +67,8 @@ case class MandrillAdapter(schemas: MandrillSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -90,7 +91,7 @@ case class MandrillAdapter(schemas: MandrillSchemas) extends Adapter { .invalidNel ) case (Some(body), _) => - payloadBodyToEvents(body) match { + payloadBodyToEvents(body, maxJsonDepth) match { case Left(str) => Monad[F].pure(str.invalidNel) case Right(list) => val _ = client @@ -133,7 +134,7 @@ case class MandrillAdapter(schemas: MandrillSchemas) extends Adapter { * @param rawEventString The encoded string from the Mandrill payload body * @return a list of single events formatted as JSONs or a Failure String */ - private[registry] def payloadBodyToEvents(rawEventString: String): Either[FailureDetails.AdapterFailure, List[Json]] = + private[registry] def payloadBodyToEvents(rawEventString: String, maxJsonDepth: Int): Either[FailureDetails.AdapterFailure, List[Json]] = for { bodyMap <- ConversionUtils .parseUrlEncodedForm(rawEventString) @@ -159,7 +160,7 @@ case class MandrillAdapter(schemas: MandrillSchemas) extends Adapter { .asLeft case Some(dStr) => JsonUtils - .extractJson(dStr) + .extractJson(dStr, maxJsonDepth) .leftMap(e => FailureDetails.AdapterFailure .NotJson("mandril_events", dStr.some, e) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MarketoAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MarketoAdapter.scala index 3333e264e..55ef5b6cf 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MarketoAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/MarketoAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -72,7 +72,8 @@ case class MarketoAdapter(schemas: MarketoSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -84,7 +85,7 @@ case class MarketoAdapter(schemas: MarketoSchemas) extends Adapter { Monad[F].pure(failure.invalidNel) case (Some(body), _) => val _ = client - val event = payloadBodyToEvent(body, payload) + val event = payloadBodyToEvent(body, payload, maxJsonDepth) Monad[F].pure(rawEventsListProcessor(List(event))) } @@ -95,10 +96,14 @@ case class MarketoAdapter(schemas: MarketoSchemas) extends Adapter { * @param payload Rest of the payload details * @return a validated JSON payload on Success, or a NEL */ - private def payloadBodyToEvent(json: String, payload: CollectorPayload): ValidatedNel[FailureDetails.AdapterFailure, RawEvent] = + private def payloadBodyToEvent( + json: String, + payload: CollectorPayload, + maxJsonDepth: Int + ): ValidatedNel[FailureDetails.AdapterFailure, RawEvent] = (for { parsed <- JU - .extractJson(json) + .extractJson(json, maxJsonDepth) .leftMap(e => FailureDetails.AdapterFailure.NotJson("body", json.some, e)) parsedConverted <- if (parsed.isObject) reformatParameters(parsed).asRight diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala index 0a5c03e0f..95bb838f8 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/OlarkAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -68,7 +68,8 @@ case class OlarkAdapter(schemas: OlarkSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -112,7 +113,7 @@ case class OlarkAdapter(schemas: OlarkSchemas) extends Adapter { case TS(bodyMap) => Monad[F].pure( (for { - event <- payloadBodyToEvent(bodyMap) + event <- payloadBodyToEvent(bodyMap, maxJsonDepth) eventType = event.hcursor.get[Json]("operators").toOption match { case Some(_) => "transcript" case _ => "offline_message" @@ -176,14 +177,14 @@ case class OlarkAdapter(schemas: OlarkSchemas) extends Adapter { * Converts a querystring payload into an event * @param bodyMap The converted map from the querystring */ - private def payloadBodyToEvent(bodyMap: Map[String, String]): Either[FailureDetails.AdapterFailure, Json] = + private def payloadBodyToEvent(bodyMap: Map[String, String], maxJsonDepth: Int): Either[FailureDetails.AdapterFailure, Json] = bodyMap.get("data") match { case None | Some("") => FailureDetails.AdapterFailure .InputData("data", none, "missing 'data' field") .asLeft case Some(json) => - JU.extractJson(json) + JU.extractJson(json, maxJsonDepth) .leftMap(e => FailureDetails.AdapterFailure.NotJson("data", json.some, e)) } } diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PagerdutyAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PagerdutyAdapter.scala index 697eef5a1..e1797283f 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PagerdutyAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PagerdutyAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -62,7 +62,8 @@ case class PagerdutyAdapter(schemas: PagerdutySchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -85,7 +86,7 @@ case class PagerdutyAdapter(schemas: PagerdutySchemas) extends Adapter { .invalidNel ) case (Some(body), _) => - payloadBodyToEvents(body) match { + payloadBodyToEvents(body, maxJsonDepth) match { case Left(str) => Monad[F].pure(str.invalidNel) case Right(list) => val _ = client @@ -126,9 +127,9 @@ case class PagerdutyAdapter(schemas: PagerdutySchemas) extends Adapter { * @param body The payload body from the PagerDuty event * @return either a Successful List of JValue JSONs or a Failure String */ - private[registry] def payloadBodyToEvents(body: String): Either[FailureDetails.AdapterFailure, List[Json]] = + private[registry] def payloadBodyToEvents(body: String, maxJsonDepth: Int): Either[FailureDetails.AdapterFailure, List[Json]] = JsonUtils - .extractJson(body) + .extractJson(body, maxJsonDepth) .leftMap(e => FailureDetails.AdapterFailure.NotJson("body", body.some, e)) .flatMap { p => p.hcursor.downField("messages").focus.flatMap(_.asArray) match { diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PingdomAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PingdomAdapter.scala index ca2b17eda..91a5c6ee4 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PingdomAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/PingdomAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -59,7 +59,8 @@ case class PingdomAdapter(schemas: PingdomSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = payload.querystring match { case Nil => @@ -75,7 +76,7 @@ case class PingdomAdapter(schemas: PingdomSchemas) extends Adapter { case Some(Some(event)) => Monad[F].pure((for { parsedEvent <- JsonUtils - .extractJson(event) + .extractJson(event, maxJsonDepth) .leftMap(e => FailureDetails.AdapterFailure .NotJson("message", event.some, e) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/RemoteAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/RemoteAdapter.scala index 16317ca28..005be7ffc 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/RemoteAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/RemoteAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -46,7 +46,8 @@ final case class RemoteAdapter[F[_]: Monad]( * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ def toRawEvents( - payload: CollectorPayload + payload: CollectorPayload, + maxJsonDepth: Int ): F[Adapted] = payload.body match { case Some(body) if body.nonEmpty => @@ -58,7 +59,7 @@ final case class RemoteAdapter[F[_]: Monad]( ) httpClient .getResponse(remoteUrl, None, None, Some(json.noSpaces), "POST") - .map(processResponse(payload, _).toValidatedNel) + .map(processResponse(payload, _, maxJsonDepth).toValidatedNel) case _ => val msg = s"empty body: not a valid remote adapter $remoteUrl payload" Monad[F].pure( @@ -71,7 +72,8 @@ final case class RemoteAdapter[F[_]: Monad]( */ def processResponse( payload: CollectorPayload, - response: Either[Throwable, String] + response: Either[Throwable, String], + maxJsonDepth: Int ): Either[FailureDetails.AdapterFailure, NonEmptyList[RawEvent]] = for { res <- response @@ -83,7 +85,7 @@ final case class RemoteAdapter[F[_]: Monad]( ) ) json <- JsonUtils - .extractJson(res) + .extractJson(res, maxJsonDepth) .leftMap(e => FailureDetails.AdapterFailure.NotJson("body", res.some, "[REMOTE_ADAPTER] " + e)) events <- json.hcursor .downField("events") diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/SendgridAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/SendgridAdapter.scala index f263dd001..798b2aa87 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/SendgridAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/SendgridAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -64,7 +64,8 @@ case class SendgridAdapter(schemas: SendgridSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -88,7 +89,7 @@ case class SendgridAdapter(schemas: SendgridSchemas) extends Adapter { ) case (Some(body), _) => val _ = client - val events = payloadBodyToEvents(body, payload) + val events = payloadBodyToEvents(body, payload, maxJsonDepth) Monad[F].pure(rawEventsListProcessor(events)) } @@ -100,8 +101,12 @@ case class SendgridAdapter(schemas: SendgridSchemas) extends Adapter { * @return a list of validated events, successes will be the corresponding raw events failures * will contain a non empty list of the reason(s) for the particular event failing */ - private def payloadBodyToEvents(body: String, payload: CollectorPayload): List[ValidatedNel[FailureDetails.AdapterFailure, RawEvent]] = - JsonUtils.extractJson(body) match { + private def payloadBodyToEvents( + body: String, + payload: CollectorPayload, + maxJsonDepth: Int + ): List[ValidatedNel[FailureDetails.AdapterFailure, RawEvent]] = + JsonUtils.extractJson(body, maxJsonDepth) match { case Right(json) => json.asArray match { case Some(array) => diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala index 84f308957..386e8bea4 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/StatusGatorAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -58,7 +58,8 @@ case class StatusGatorAdapter(schemas: StatusGatorSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala index b2a3b86eb..7123b0c27 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UnbounceAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -62,7 +62,8 @@ case class UnbounceAdapter(schemas: UnbounceSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -108,7 +109,7 @@ case class UnbounceAdapter(schemas: UnbounceSchemas) extends Adapter { case TS(bodyMap) => Monad[F].pure( ( - payloadBodyToEvent(bodyMap).toValidatedNel, + payloadBodyToEvent(bodyMap, maxJsonDepth).toValidatedNel, lookupSchema(Some("form_post"), ContextSchema).toValidatedNel ).mapN { (event, schema) => NonEmptyList.one( @@ -125,7 +126,7 @@ case class UnbounceAdapter(schemas: UnbounceSchemas) extends Adapter { } } - private def payloadBodyToEvent(bodyMap: Map[String, String]): Either[FailureDetails.AdapterFailure, Json] = + private def payloadBodyToEvent(bodyMap: Map[String, String], maxJsonDepth: Int): Either[FailureDetails.AdapterFailure, Json] = ( bodyMap.get("page_id"), bodyMap.get("page_name"), @@ -159,7 +160,7 @@ case class UnbounceAdapter(schemas: UnbounceSchemas) extends Adapter { .asLeft case (Some(_), Some(_), Some(_), Some(_), Some(dataJson)) => val event = (bodyMap - "data.json" - "data.xml").toList - JU.extractJson(dataJson) + JU.extractJson(dataJson, maxJsonDepth) .map { dJs => val js = Json .obj( diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UrbanAirshipAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UrbanAirshipAdapter.scala index 8d19283c9..67350a0cf 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UrbanAirshipAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/UrbanAirshipAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -69,7 +69,8 @@ case class UrbanAirshipAdapter(schemas: UrbanAirshipSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -88,7 +89,7 @@ case class UrbanAirshipAdapter(schemas: UrbanAirshipSchemas) extends Adapter { ) case (Some(body), _) => val _ = client - val event = payloadBodyToEvent(body, payload) + val event = payloadBodyToEvent(body, payload, maxJsonDepth) Monad[F].pure(rawEventsListProcessor(List(event))) } @@ -99,7 +100,11 @@ case class UrbanAirshipAdapter(schemas: UrbanAirshipSchemas) extends Adapter { * @param payload other payload details * @return a validated event - a success is the RawEvent, failures will contain the reasons */ - private def payloadBodyToEvent(bodyJson: String, payload: CollectorPayload): ValidatedNel[FailureDetails.AdapterFailure, RawEvent] = { + private def payloadBodyToEvent( + bodyJson: String, + payload: CollectorPayload, + maxJsonDepth: Int + ): ValidatedNel[FailureDetails.AdapterFailure, RawEvent] = { def toTtmFormat(jsonTimestamp: String) = "%d".format(new DateTime(jsonTimestamp).getMillis) @@ -110,7 +115,7 @@ case class UrbanAirshipAdapter(schemas: UrbanAirshipSchemas) extends Adapter { s"could not extract '$field': ${e.getMessage}" ) - JsonUtils.extractJson(bodyJson) match { + JsonUtils.extractJson(bodyJson, maxJsonDepth) match { case Right(json) => val cursor = json.hcursor val eventType = cursor.get[String]("type").toOption diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/VeroAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/VeroAdapter.scala index 0c912fdac..7dc436a9e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/VeroAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/VeroAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -56,7 +56,8 @@ case class VeroAdapter(schemas: VeroSchemas) extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = (payload.body, payload.contentType) match { case (None, _) => @@ -68,7 +69,7 @@ case class VeroAdapter(schemas: VeroSchemas) extends Adapter { Monad[F].pure(failure.invalidNel) case (Some(body), _) => val _ = client - val event = payloadBodyToEvent(body, payload) + val event = payloadBodyToEvent(body, payload, maxJsonDepth) Monad[F].pure(rawEventsListProcessor(List(event.toValidatedNel))) } @@ -79,10 +80,14 @@ case class VeroAdapter(schemas: VeroSchemas) extends Adapter { * @param payload The details of the payload * @return a Validation boxing either a NEL of RawEvents on Success, or a NEL of Failure Strings */ - private def payloadBodyToEvent(json: String, payload: CollectorPayload): Either[FailureDetails.AdapterFailure, RawEvent] = + private def payloadBodyToEvent( + json: String, + payload: CollectorPayload, + maxJsonDepth: Int + ): Either[FailureDetails.AdapterFailure, RawEvent] = for { parsed <- JsonUtils - .extractJson(json) + .extractJson(json, maxJsonDepth) .leftMap(e => FailureDetails.AdapterFailure.NotJson("body", json.some, e)) eventType <- parsed.hcursor .get[String]("type") diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala index 588f27022..0b65d15e3 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/RedirectAdapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -73,7 +73,8 @@ object RedirectAdapter extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = { val _ = client val originalParams = toMap(payload.querystring) @@ -96,8 +97,8 @@ object RedirectAdapter extends Adapter { (originalParams.get("cx"), originalParams.get("co")) match { case (None, None) => newCo.asRight case (None, Some(Some(co))) if co == "" => newCo.asRight - case (None, Some(Some(co))) => addToExistingCo(json, co).map(str => Map("co" -> str)) - case (Some(Some(cx)), _) => addToExistingCx(json, cx).map(str => Map("cx" -> str)) + case (None, Some(Some(co))) => addToExistingCo(json, co, maxJsonDepth).map(str => Map("co" -> str)) + case (Some(Some(cx)), _) => addToExistingCx(json, cx, maxJsonDepth).map(str => Map("cx" -> str)) } } else // Add URI redirect as an unstructured event @@ -152,11 +153,12 @@ object RedirectAdapter extends Adapter { */ private def addToExistingCo( newContext: SelfDescribingData[Json], - existing: String + existing: String, + maxJsonDepth: Int ): Either[FailureDetails.TrackerProtocolViolation, String] = for { json <- JU - .extractJson(existing) // co|cx + .extractJson(existing, maxJsonDepth) // co|cx .leftMap(e => FailureDetails.TrackerProtocolViolation .NotJson("co|cx", existing.some, e) @@ -178,7 +180,8 @@ object RedirectAdapter extends Adapter { */ private def addToExistingCx( newContext: SelfDescribingData[Json], - existing: String + existing: String, + maxJsonDepth: Int ): Either[FailureDetails.TrackerProtocolViolation, String] = for { decoded <- CU @@ -187,7 +190,7 @@ object RedirectAdapter extends Adapter { FailureDetails.TrackerProtocolViolation .InputData("cx", existing.some, e) ) - added <- addToExistingCo(newContext, decoded) + added <- addToExistingCo(newContext, decoded, maxJsonDepth) recoded = CU.encodeBase64Url(added) } yield recoded diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala index 7c0b0d2e0..b7b68cb23 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp1Adapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -39,7 +39,8 @@ object Tp1Adapter extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = { val _ = client val params = toMap(payload.querystring) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala index 0fc2d8810..056ac6ac9 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/adapters/registry/snowplow/Tp2Adapter.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -59,7 +59,8 @@ object Tp2Adapter extends Adapter { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): F[Adapted] = { val qsParams = toMap(payload.querystring) @@ -107,7 +108,7 @@ object Tp2Adapter extends Adapter { case (None, None) => Monad[F].pure(NonEmptyList.one(qsParams).valid) case (Some(bdy), Some(_)) => // Build our NEL of parameters (for { - json <- extractAndValidateJson(PayloadDataSchema, bdy, client, registryLookup) + json <- extractAndValidateJson(PayloadDataSchema, bdy, client, registryLookup, maxJsonDepth) nel <- EitherT.fromEither[F](toParametersNel(json, qsParams)) } yield nel).toValidated } @@ -213,12 +214,13 @@ object Tp2Adapter extends Adapter { schemaCriterion: SchemaCriterion, instance: String, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): EitherT[F, NonEmptyList[FailureDetails.TrackerProtocolViolation], Json] = { implicit val rl = registryLookup (for { j <- EitherT.fromEither[F]( - JU.extractJson(instance) + JU.extractJson(instance, maxJsonDepth) .leftMap(e => NonEmptyList.one( FailureDetails.TrackerProtocolViolation diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFields.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFields.scala index 0d4c4b9ab..5e060cc7a 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFields.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFields.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFieldsLengthValidator.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFieldsLengthValidator.scala index b995e94b6..cdb927c51 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFieldsLengthValidator.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/AtomicFieldsLengthValidator.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala index b0a541370..86893c226 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/ClientEnrichments.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala index 1bacdc4e3..25f650b82 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentManager.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -69,7 +69,8 @@ object EnrichmentManager { invalidCount: F[Unit], registryLookup: RegistryLookup[F], atomicFields: AtomicFields, - emitIncomplete: Boolean + emitIncomplete: Boolean, + maxJsonDepth: Int ): IorT[F, BadRow, EnrichedEvent] = { def enrich(enriched: EnrichedEvent): IorT[F, NonEmptyList[NonEmptyList[Failure]], List[SelfDescribingData[Json]]] = for { @@ -79,7 +80,8 @@ object EnrichmentManager { etlTstamp, processor, client, - registryLookup + registryLookup, + maxJsonDepth ) .leftMap(NonEmptyList.one) .possiblyExitingEarly(emitIncomplete) @@ -93,7 +95,8 @@ object EnrichmentManager { enriched, extractResult.contexts, extractResult.unstructEvent, - featureFlags.legacyEnrichmentOrder + featureFlags.legacyEnrichmentOrder, + maxJsonDepth ) .leftMap(NonEmptyList.one) .possiblyExitingEarly(emitIncomplete) @@ -179,13 +182,14 @@ object EnrichmentManager { etlTstamp: DateTime, processor: Processor, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): IorT[F, NonEmptyList[Failure], IgluUtils.EventExtractResult] = for { _ <- setupEnrichedEvent[F](raw, enrichedEvent, etlTstamp, processor) .leftMap(NonEmptyList.one) extract <- IgluUtils - .extractAndValidateInputJsons(enrichedEvent, client, registryLookup) + .extractAndValidateInputJsons(enrichedEvent, client, registryLookup, maxJsonDepth) .leftMap { l: NonEmptyList[Failure] => l } } yield extract @@ -201,10 +205,11 @@ object EnrichmentManager { enriched: EnrichedEvent, inputContexts: List[SelfDescribingData[Json]], unstructEvent: Option[SelfDescribingData[Json]], - legacyOrder: Boolean + legacyOrder: Boolean, + maxJsonDepth: Int ): IorT[F, NonEmptyList[Failure], List[SelfDescribingData[Json]]] = IorT { - accState(registry, raw, inputContexts, unstructEvent, legacyOrder) + accState(registry, raw, inputContexts, unstructEvent, legacyOrder, maxJsonDepth) .runS(Accumulation(enriched, Nil, Nil)) .map { case Accumulation(_, failures, contexts) => @@ -283,7 +288,8 @@ object EnrichmentManager { raw: RawEvent, inputContexts: List[SelfDescribingData[Json]], unstructEvent: Option[SelfDescribingData[Json]], - legacyOrder: Boolean + legacyOrder: Boolean, + maxJsonDepth: Int ): EStateT[F, Unit] = { val getCookieContexts = headerContexts[F, CookieExtractorEnrichment]( raw.context.headers, @@ -316,7 +322,9 @@ object EnrichmentManager { _ <- getHttpHeaderContexts // Execute header extractor enrichment _ <- getYauaaContext[F](registry.yauaa, raw.context.headers) // Runs YAUAA enrichment (gets info thanks to user agent) _ <- extractSchemaFields[F](unstructEvent) // Extract the event vendor/name/format/version - _ <- registry.javascriptScript.traverse(getJsScript[F](_)) // Execute the JavaScript scripting enrichment + _ <- registry.javascriptScript.traverse( // Execute the JavaScript scripting enrichment + getJsScript[F](_, raw.context.headers, maxJsonDepth) + ) _ <- getCurrency[F](raw.context.timestamp, registry.currencyConversion) // Finalize the currency conversion _ <- getWeatherContext[F](registry.weather) // Fetch weather context _ <- geoLocation[F](registry.ipLookups) // Execute IP lookup enrichment @@ -347,7 +355,9 @@ object EnrichmentManager { _ <- getYauaaContext[F](registry.yauaa, raw.context.headers) // Runs YAUAA enrichment (gets info thanks to user agent) _ <- extractSchemaFields[F](unstructEvent) // Extract the event vendor/name/format/version _ <- geoLocation[F](registry.ipLookups) // Execute IP lookup enrichment - _ <- registry.javascriptScript.traverse(getJsScript[F](_)) // Execute the JavaScript scripting enrichment + _ <- registry.javascriptScript.traverse( // Execute the JavaScript scripting enrichment + getJsScript[F](_, raw.context.headers, maxJsonDepth) + ) _ <- sqlContexts // Derive some contexts with custom SQL Query enrichment _ <- apiContexts // Derive some contexts with custom API Request enrichment _ <- anonIp[F](registry.anonIp) // Anonymize the IP @@ -788,12 +798,14 @@ object EnrichmentManager { // Execute the JavaScript scripting enrichment def getJsScript[F[_]: Applicative]( - javascriptScript: JavascriptScriptEnrichment + javascriptScript: JavascriptScriptEnrichment, + headers: List[String], + maxJsonDepth: Int ): EStateT[F, Unit] = EStateT.fromEither { case (event, derivedContexts) => ME.formatContexts(derivedContexts).foreach(c => event.derived_contexts = c) - javascriptScript.process(event).leftMap(NonEmptyList.one) + javascriptScript.process(event, headers, maxJsonDepth).leftMap(NonEmptyList.one) } def headerContexts[F[_]: Applicative, A]( diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala index a9512b24d..cdb85a2e0 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EnrichmentRegistry.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EventEnrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EventEnrichments.scala index deb5f812a..1a46e0fec 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EventEnrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/EventEnrichments.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Failure.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Failure.scala index 3e0aa44cb..4270271da 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Failure.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Failure.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/MiscEnrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/MiscEnrichments.scala index 8754eb477..4479c0894 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/MiscEnrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/MiscEnrichments.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/SchemaEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/SchemaEnrichment.scala index 38a304763..22ca90191 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/SchemaEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/SchemaEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Transform.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Transform.scala index c065c1b79..7e290b436 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Transform.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/Transform.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala index be2342ae9..f2763c356 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/AnonIpEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CachingEvaluator.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CachingEvaluator.scala index cce039142..22cb05a8e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CachingEvaluator.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CachingEvaluator.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala index b7859920a..334e3a912 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CampaignAttributionEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala index 6136d1955..5848a21be 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CookieExtractorEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CrossNavigationEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CrossNavigationEnrichment.scala index 54083d769..f30ce4e23 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CrossNavigationEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CrossNavigationEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala index 95b07ab91..d3a54369d 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/CurrencyConversionEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala index 528968cbd..5f244cf82 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EnrichmentConf.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala index 755bc03b2..2bfa48a40 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/EventFingerprintEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala index db5d7d088..2442e9378 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/HttpHeaderExtractorEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala index 1936815ac..3f424b481 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IabEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala index c4a055c20..2fbb05e51 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/IpLookupsEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala index 4578b047a..5824d8276 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/JavascriptScriptEnrichment.scala @@ -3,18 +3,19 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ package com.snowplowanalytics.snowplow.enrich.common.enrichments.registry +import scala.collection.JavaConverters._ + import cats.data.{NonEmptyList, ValidatedNel} import cats.implicits._ import io.circe._ -import io.circe.parser._ import io.circe.syntax._ import javax.script._ @@ -26,7 +27,7 @@ import com.snowplowanalytics.snowplow.badrows.FailureDetails import com.snowplowanalytics.snowplow.enrich.common.enrichments.registry.EnrichmentConf.JavascriptScriptConf import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent -import com.snowplowanalytics.snowplow.enrich.common.utils.{CirceUtils, ConversionUtils} +import com.snowplowanalytics.snowplow.enrich.common.utils.{CirceUtils, ConversionUtils, JsonUtils} object JavascriptScriptEnrichment extends ParseableEnrichment { override val supportedSchema = @@ -73,8 +74,8 @@ final case class JavascriptScriptEnrichment( private val stringified = rawFunction + s""" var getJavascriptContexts = function() { const params = ${params.asJson.noSpaces}; - return function(event) { - const result = process(event, params); + return function(event, headers) { + const result = process(event, params, headers); if (result == null) { return "[]" } else { @@ -95,15 +96,19 @@ final case class JavascriptScriptEnrichment( * The event can be updated in-place by the JS function. * @return either a JSON array of contexts on Success, or an error String on Failure */ - def process(event: EnrichedEvent): Either[FailureDetails.EnrichmentFailure, List[SelfDescribingData[Json]]] = + def process( + event: EnrichedEvent, + headers: List[String], + maxJsonDepth: Int + ): Either[FailureDetails.EnrichmentFailure, List[SelfDescribingData[Json]]] = invocable .flatMap(_ => Either - .catchNonFatal(engine.invokeFunction("getJavascriptContexts", event).asInstanceOf[String]) + .catchNonFatal(engine.invokeFunction("getJavascriptContexts", event, headers.asJava).asInstanceOf[String]) .leftMap(e => s"Error during execution of JavaScript function: [${e.getMessage}]") ) .flatMap(contexts => - parse(contexts) match { + JsonUtils.extractJson(contexts, maxJsonDepth) match { case Right(json) => json.asArray match { case Some(array) => @@ -127,7 +132,7 @@ final case class JavascriptScriptEnrichment( Left(s"Output of JavaScript function [$json] could be parsed as JSON but is not read as an array") } case Left(err) => - Left(s"Could not parse output JSON of Javascript function. Error: [${err.getMessage}]") + Left(s"Could not parse output JSON of Javascript function. Error: [$err]") } ) .leftMap(errorMsg => FailureDetails.EnrichmentFailure(enrichmentInfo, FailureDetails.EnrichmentFailureMessage.Simple(errorMsg))) diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala index ce9cb4df1..6f0ee2b3d 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/RefererParserEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala index 950d4526b..fe6c8344c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UaParserEnrichment.scala @@ -2,8 +2,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala index 88583201c..4eafc9d22 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/UserAgentUtilsEnrichment.scala @@ -2,8 +2,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala index 6d9b3be12..20412a137 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/WeatherEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala index 55127907c..a1f6d4c94 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/YauaaEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala index 90126baeb..cc76edb92 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/ApiRequestEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Cache.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Cache.scala index e1fd520b8..89428b72e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Cache.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Cache.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Errors.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Errors.scala index 68e43a0a9..88834f49f 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Errors.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Errors.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/HttpApi.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/HttpApi.scala index 483a64595..6bcc01672 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/HttpApi.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/HttpApi.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala index 851c5a34c..1f5eb0f27 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Input.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Output.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Output.scala index 2aee62888..d342ef8cd 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Output.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/apirequest/Output.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala index 7c2eec2b2..92c4ea8ce 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/enrichments.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Mutators.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Mutators.scala index 45a23f4dc..0f627a0f0 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Mutators.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/Mutators.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala index 955bf6ac7..8f65dbd5e 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/PiiPseudonymizerEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/package.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/package.scala index 36c5a478d..8b9f37890 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/package.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/package.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/serializers.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/serializers.scala index de86179a4..cf5c674c3 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/serializers.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/pii/serializers.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/DbExecutor.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/DbExecutor.scala index 567c586f0..26676ac8c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/DbExecutor.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/DbExecutor.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Errors.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Errors.scala index 7f2ca1c74..6615c3dfd 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Errors.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Errors.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Input.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Input.scala index 921cceb66..1107ad32d 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Input.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Input.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/JsonOutput.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/JsonOutput.scala index 406d9aa7e..a5d5a0543 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/JsonOutput.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/JsonOutput.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Output.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Output.scala index 5620edb8d..2521b59c3 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Output.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Output.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Rdbms.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Rdbms.scala index 308610660..c095cdf2c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Rdbms.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/Rdbms.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala index dad1a118a..652a82332 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/SqlQueryEnrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/package.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/package.scala index 5d33bd806..7d0c13ddd 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/package.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/registry/sqlquery/package.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/web/PageEnrichments.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/web/PageEnrichments.scala index b7c6838cf..dabca9d87 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/web/PageEnrichments.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/enrichments/web/PageEnrichments.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CollectorPayload.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CollectorPayload.scala index 3ba901d9b..f5fa6ab75 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CollectorPayload.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/CollectorPayload.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/IpAddressExtractor.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/IpAddressExtractor.scala index 625111184..ce86eaae4 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/IpAddressExtractor.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/IpAddressExtractor.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala index 8df5099d0..d82427313 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/Loader.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala index 55e9099fd..7993b0806 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/loaders/ThriftLoader.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -36,7 +36,8 @@ import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, ParseError = /** Loader for Thrift SnowplowRawEvent objects. */ object ThriftLoader extends Loader[Array[Byte]] { - private val thriftDeserializer = new TDeserializer + private val thriftDeserializer: ThreadLocal[TDeserializer] = + ThreadLocal.withInitial(() => new TDeserializer) private[loaders] val ExpectedSchema = SchemaCriterion("com.snowplowanalytics.snowplow", "CollectorPayload", "thrift", 1, 0) @@ -111,13 +112,13 @@ object ThriftLoader extends Loader[Array[Byte]] { private def extractSchema(tryBase64Decoding: Boolean, line: Array[Byte]): (SchemaSniffer, Array[Byte]) = try { val schema = new SchemaSniffer() - this.synchronized(thriftDeserializer.deserialize(schema, line)) + thriftDeserializer.get.deserialize(schema, line) (schema, line) } catch { case NonFatal(_) if tryBase64Decoding => val base64Decoded = Base64.decodeBase64(line) val schema = new SchemaSniffer() - this.synchronized(thriftDeserializer.deserialize(schema, base64Decoded)) + thriftDeserializer.get.deserialize(schema, base64Decoded) (schema, base64Decoded) } @@ -132,12 +133,10 @@ object ThriftLoader extends Loader[Array[Byte]] { */ private def convertSchema1(line: Array[Byte]): ValidatedNel[FailureDetails.CPFormatViolationMessage, Option[CollectorPayload]] = { val collectorPayload = new CollectorPayload1 - this.synchronized { - thriftDeserializer.deserialize( - collectorPayload, - line - ) - } + thriftDeserializer.get.deserialize( + collectorPayload, + line + ) val querystring = parseQuerystring( Option(collectorPayload.querystring), @@ -195,12 +194,10 @@ object ThriftLoader extends Loader[Array[Byte]] { */ private def convertOldSchema(line: Array[Byte]): ValidatedNel[FailureDetails.CPFormatViolationMessage, Option[CollectorPayload]] = { val snowplowRawEvent = new SnowplowRawEvent() - this.synchronized { - thriftDeserializer.deserialize( - snowplowRawEvent, - line - ) - } + thriftDeserializer.get.deserialize( + snowplowRawEvent, + line + ) val querystring = parseQuerystring( Option(snowplowRawEvent.payload.data), diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/outputs/EnrichedEvent.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/outputs/EnrichedEvent.scala index 49857ea0c..713e29142 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/outputs/EnrichedEvent.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/outputs/EnrichedEvent.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/AtomicError.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/AtomicError.scala index ab360dcc7..91fdef413 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/AtomicError.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/AtomicError.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/CirceUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/CirceUtils.scala index dff76cedc..cc1bb0334 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/CirceUtils.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/CirceUtils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala index c791a9123..a0fc71e8b 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ConversionUtils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/HttpClient.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/HttpClient.scala index b6c911443..1c4c8c1e8 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/HttpClient.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/HttpClient.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala index 3d058a136..dda8a54e2 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/IgluUtils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -52,11 +52,12 @@ object IgluUtils { def extractAndValidateInputJsons[F[_]: Monad: Clock]( enriched: EnrichedEvent, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): IorT[F, NonEmptyList[Failure.SchemaViolation], EventExtractResult] = for { - contexts <- extractAndValidateInputContexts(enriched, client, registryLookup) - unstruct <- extractAndValidateUnstructEvent(enriched, client, registryLookup) + contexts <- extractAndValidateInputContexts(enriched, client, registryLookup, maxJsonDepth) + unstruct <- extractAndValidateUnstructEvent(enriched, client, registryLookup, maxJsonDepth) } yield { val validationInfoContexts = (contexts.flatMap(_.validationInfo) ::: unstruct.flatMap(_.validationInfo).toList).distinct .map(_.toSdj) @@ -79,6 +80,7 @@ object IgluUtils { enriched: EnrichedEvent, client: IgluCirceClient[F], registryLookup: RegistryLookup[F], + maxJsonDepth: Int, field: String = "unstruct", criterion: SchemaCriterion = SchemaCriterion("com.snowplowanalytics.snowplow", "unstruct_event", "jsonschema", 1, 0) ): IorT[F, NonEmptyList[Failure.SchemaViolation], Option[SdjExtractResult]] = @@ -86,7 +88,7 @@ object IgluUtils { case Some(rawUnstructEvent) => val iorT = for { // Validate input Json string and extract unstructured event - unstruct <- extractInputData(rawUnstructEvent, field, criterion, client, registryLookup) + unstruct <- extractInputData(rawUnstructEvent, field, criterion, client, registryLookup, maxJsonDepth) .leftMap(NonEmptyList.one) .toIor // Parse Json unstructured event as SelfDescribingData[Json] @@ -110,6 +112,7 @@ object IgluUtils { enriched: EnrichedEvent, client: IgluCirceClient[F], registryLookup: RegistryLookup[F], + maxJsonDepth: Int, field: String = "contexts", criterion: SchemaCriterion = SchemaCriterion("com.snowplowanalytics.snowplow", "contexts", "jsonschema", 1, 0) ): IorT[F, NonEmptyList[Failure.SchemaViolation], List[SdjExtractResult]] = @@ -117,7 +120,7 @@ object IgluUtils { case Some(rawContexts) => val iorT = for { // Validate input Json string and extract contexts - contexts <- extractInputData(rawContexts, field, criterion, client, registryLookup) + contexts <- extractInputData(rawContexts, field, criterion, client, registryLookup, maxJsonDepth) .map(_.asArray.get.toList) // .get OK because SDJ wrapping the contexts valid .leftMap(NonEmptyList.one) .toIor @@ -167,12 +170,13 @@ object IgluUtils { field: String, // to put in the bad row expectedCriterion: SchemaCriterion, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ): EitherT[F, Failure.SchemaViolation, Json] = for { // Parse Json string with the SDJ json <- JsonUtils - .extractJson(rawJson) + .extractJson(rawJson, maxJsonDepth) .leftMap(e => Failure.SchemaViolation( schemaViolation = FailureDetails.SchemaViolation.NotJson(field, rawJson.some, e), diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala index 00e2dd004..802d7b464 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonPath.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonUtils.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonUtils.scala index 545ad1ae3..7794b8e9c 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonUtils.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/JsonUtils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -16,6 +16,7 @@ import cats.data.NonEmptyList import cats.syntax.either._ import io.circe.Json +import io.circe.syntax._ import org.joda.time.{DateTime, DateTimeZone} import org.joda.time.format.{DateTimeFormat, DateTimeFormatter} @@ -119,12 +120,29 @@ object JsonUtils { /** * Converts a JSON string into an EIther[String, Json] * @param instance The JSON string to parse + * @param maxJsonDepth The maximum allowed JSON depth * @return either an error String or the extracted Json */ - def extractJson(instance: String): Either[String, Json] = - io.circe.parser - .parse(instance) - .leftMap(e => s"invalid json: ${e.message}") + def extractJson(instance: String, maxJsonDepth: Int): Either[String, Json] = + for { + json <- io.circe.parser + .parse(instance) + .leftMap(e => s"invalid json: ${e.message}") + _ <- if (checkIfExceedMaxDepth(json, maxJsonDepth)) Left("invalid json: maximum allowed JSON depth exceeded") + else Right(json) + } yield json + + private def checkIfExceedMaxDepth(json: Json, maxJsonDepth: Int): Boolean = + if (maxJsonDepth <= 0) true + else + json.fold( + jsonNull = false, + jsonBoolean = _ => false, + jsonNumber = _ => false, + jsonString = _ => false, + jsonArray = _.exists(checkIfExceedMaxDepth(_, maxJsonDepth - 1)), + jsonObject = _.toList.exists { case (_, j) => checkIfExceedMaxDepth(j.asJson, maxJsonDepth - 1) } + ) /** * Strips the instance information from a Jackson diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala index de3d45511..532ce53d9 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/MapTransformer.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ShiftExecution.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ShiftExecution.scala index 23b1bfe0e..c6b4204a8 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ShiftExecution.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/common/utils/ShiftExecution.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/package.scala b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/package.scala index e96b9c56d..11359601d 100644 --- a/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/package.scala +++ b/modules/common/src/main/scala/com.snowplowanalytics.snowplow.enrich/package.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonDeserializer.scala b/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonDeserializer.scala index d6824a3f2..f2ed39245 100644 --- a/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonDeserializer.scala +++ b/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonDeserializer.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -59,7 +59,7 @@ private[jackson] final class CirceJsonDeserializer(klass: Class[_]) extends Json case JsonTokenId.ID_FIELD_NAME => parserContext match { - case (c: ReadingMap) :: stack => (None, c.setField(jp.getCurrentName) +: stack) + case (c: ReadingMap) :: stack => (None, c.setField(jp.currentName) +: stack) case _ => throw new IllegalStateException("Jackson read a String field name but parser context is not a json object") } diff --git a/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonModule.scala b/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonModule.scala index 81039abd6..45f87a691 100644 --- a/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonModule.scala +++ b/modules/common/src/main/scala/io/circe/jackson/enrich/CirceJsonModule.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/main/scala/io/circe/jackson/enrich/package.scala b/modules/common/src/main/scala/io/circe/jackson/enrich/package.scala index 5c80494b6..9f76694b3 100644 --- a/modules/common/src/main/scala/io/circe/jackson/enrich/package.scala +++ b/modules/common/src/main/scala/io/circe/jackson/enrich/package.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/AcceptInvalid.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/AcceptInvalid.scala index c83d4099c..fb5e93dcb 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/AcceptInvalid.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/AcceptInvalid.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/EtlPipelineSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/EtlPipelineSpec.scala index b4baad14f..3692915e5 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/EtlPipelineSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/EtlPipelineSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -53,7 +53,7 @@ class EtlPipelineSpec extends Specification with ValidatedMatchers with CatsEffe ) val enrichmentReg = EnrichmentRegistry[IO]() val igluCentral = Registry.IgluCentral - def igluClient = IgluCirceClient.fromResolver[IO](Resolver(List(igluCentral), None), cacheSize = 0) + def igluClient = IgluCirceClient.fromResolver[IO](Resolver[IO](List(igluCentral), None), cacheSize = 0, maxJsonDepth = 40) val processor = Processor("sce-test-suite", "1.0.0") val dateTime = DateTime.now() @@ -73,7 +73,8 @@ class EtlPipelineSpec extends Specification with ValidatedMatchers with CatsEffe IO.unit, SpecHelpers.registryLookup, AtomicFields.from(Map.empty), - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) } yield output must be like { case Ior.Right(_) :: Ior.Left(_) :: Ior.Left(_) :: Ior.Left(_) :: Nil => ok @@ -101,7 +102,8 @@ class EtlPipelineSpec extends Specification with ValidatedMatchers with CatsEffe IO.unit, SpecHelpers.registryLookup, AtomicFields.from(Map.empty), - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) } yield output must be like { case Ior.Right(_) :: Nil => ok @@ -124,7 +126,8 @@ class EtlPipelineSpec extends Specification with ValidatedMatchers with CatsEffe IO.unit, SpecHelpers.registryLookup, AtomicFields.from(Map.empty), - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) } yield output must be like { case Ior.Left(_: BadRow.CPFormatViolation) :: Nil => ok @@ -147,7 +150,8 @@ class EtlPipelineSpec extends Specification with ValidatedMatchers with CatsEffe IO.unit, SpecHelpers.registryLookup, AtomicFields.from(Map.empty), - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) } yield output must beEqualTo(Nil) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala index 5a3821f4b..81b0deeec 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/SpecHelpers.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -49,6 +49,8 @@ object SpecHelpers extends CatsEffect { val StaticTime = 1599750938180L + val DefaultMaxJsonDepth = 40 + // Standard Iglu configuration private val igluConfig = json"""{ "schema": "iglu:com.snowplowanalytics.iglu/resolver-config/jsonschema/1-0-0", @@ -81,11 +83,19 @@ object SpecHelpers extends CatsEffect { /** Builds an Iglu client from the above Iglu configuration. */ val client: IgluCirceClient[IO] = IgluCirceClient - .parseDefault[IO](igluConfig) + .parseDefault[IO](igluConfig, maxJsonDepth = 40) .value .unsafeRunSync() .getOrElse(throw new RuntimeException("invalid resolver configuration")) + /** Builds an Iglu client with given max JSON depth. */ + def client(maxJsonDepth: Int): IgluCirceClient[IO] = + IgluCirceClient + .parseDefault[IO](igluConfig, maxJsonDepth) + .value + .unsafeRunSync() + .getOrElse(throw new RuntimeException("invalid resolver configuration")) + val registryLookup = JavaNetRegistryLookup.ioLookupInstance[IO] val blockingEC = ExecutionContext.fromExecutorService(Executors.newCachedThreadPool) @@ -124,7 +134,7 @@ object SpecHelpers extends CatsEffect { /** Parse a string containing a SDJ as [[SelfDescribingData]] */ def jsonStringToSDJ(rawJson: String): Either[String, SelfDescribingData[Json]] = JsonUtils - .extractJson(rawJson) + .extractJson(rawJson, DefaultMaxJsonDepth) .leftMap(err => s"Can't parse [$rawJson] as Json, error: [$err]") .flatMap(SelfDescribingData.parse[Json]) .leftMap(err => s"Can't parse Json [$rawJson] as as SelfDescribingData, error: [$err]") @@ -167,7 +177,7 @@ object SpecHelpers extends CatsEffect { Resource.make(filesCleanup(files))(_ => filesCleanup(files)) def createIgluClient(registries: List[Registry]): IO[IgluCirceClient[IO]] = - IgluCirceClient.fromResolver[IO](Resolver(registries, None), cacheSize = 0) + IgluCirceClient.fromResolver[IO](Resolver[IO](registries, None), cacheSize = 0, maxJsonDepth = 40) val emitIncomplete = false diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/AdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/AdapterSpec.scala index 3810b1350..54259933b 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/AdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/AdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -56,7 +56,8 @@ class AdapterSpec extends Specification with DataTables with ValidatedMatchers { override def toRawEvents[F[_]: Monad: Clock]( payload: CollectorPayload, client: IgluCirceClient[F], - registryLookup: RegistryLookup[F] + registryLookup: RegistryLookup[F], + maxJsonDepth: Int ) = Monad[F].pure( FailureDetails.AdapterFailure diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CallrailAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CallrailAdapterSpec.scala index 4e6e8080e..866bd742c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CallrailAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CallrailAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -145,7 +145,7 @@ class CallrailAdapterSpec extends Specification with DataTables with ValidatedMa |}""".stripMargin.replaceAll("[\n\r]", "") adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -166,7 +166,7 @@ class CallrailAdapterSpec extends Specification with DataTables with ValidatedMa val payload = CollectorPayload(Shared.api, params, None, None, Shared.source, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CloudfrontAccessLogAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CloudfrontAccessLogAdapterSpec.scala index 52160ded8..0e17c6eda 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CloudfrontAccessLogAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/CloudfrontAccessLogAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -109,7 +109,11 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with loader .toCollectorPayload(input, processor) - .traverse(_.traverse(adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup))) + .traverse( + _.traverse( + adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + ) + ) .map( _ must beValid( Some( @@ -159,7 +163,11 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with loader .toCollectorPayload(input, processor) - .traverse(_.traverse(adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup))) + .traverse( + _.traverse( + adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + ) + ) .map( _ must beValid( Some( @@ -212,7 +220,11 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with loader .toCollectorPayload(input, processor) - .traverse(_.traverse(adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup))) + .traverse( + _.traverse( + adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + ) + ) .map( _ must beValid( Some( @@ -266,7 +278,11 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with loader .toCollectorPayload(input, processor) - .traverse(_.traverse(adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup))) + .traverse( + _.traverse( + adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + ) + ) .map( _ must beValid( Some( @@ -324,7 +340,11 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with loader .toCollectorPayload(input, processor) - .traverse(_.traverse(adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup))) + .traverse( + _.traverse( + adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + ) + ) .map( _ must beValid( Some( @@ -383,7 +403,11 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with loader .toCollectorPayload(input, processor) - .traverse(_.traverse(adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup))) + .traverse( + _.traverse( + adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + ) + ) .map( _ must beValid( Some( @@ -444,7 +468,11 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with loader .toCollectorPayload(input, processor) - .traverse(_.traverse(adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup))) + .traverse( + _.traverse( + adapterWithDefaultSchemas.toRawEvents(_, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + ) + ) .map( _ must beValid( Some( @@ -477,7 +505,7 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList @@ -505,7 +533,7 @@ class CloudfrontAccessLogAdapterSpec extends Specification with DataTables with ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.of( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala index d08072084..e9ef66026 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/GoogleAnalyticsAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -77,7 +77,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali def e1 = { val payload = CollectorPayload(api, Nil, None, None, source, context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -91,7 +91,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val body = "dl=docloc" val payload = CollectorPayload(api, Nil, None, body.some, source, context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -109,7 +109,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val body = "t=unknown&dl=docloc" val payload = CollectorPayload(api, Nil, None, body.some, source, context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.of( @@ -148,7 +148,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val expectedParams = static ++ Map("ue_pr" -> expectedJson, "co" -> expectedCO).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -180,7 +180,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val expectedParams = static ++ Map("ue_pr" -> expectedUE, "co" -> expectedCO).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -208,7 +208,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val expectedParams = static ++ Map("ue_pr" -> expectedUE, "co" -> expectedCO, "ip" -> "ip").toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -246,7 +246,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali ).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -273,7 +273,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val expectedParams = static ++ Map("ue_pr" -> expectedUE, "co" -> expectedCO).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -308,7 +308,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali ).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -338,7 +338,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val expectedParams = static ++ Map("ue_pr" -> expectedUE, "co" -> expectedCO).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -365,7 +365,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val expectedParams = static ++ Map("ue_pr" -> expectedUE, "co" -> expectedCO).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -401,7 +401,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val expectedParams = static ++ Map("ue_pr" -> expectedUE, "co" -> expectedCO).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -431,7 +431,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val expectedParams = static ++ Map("ue_pr" -> expectedUE, "co" -> expectedCO).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } @@ -459,7 +459,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali val event = RawEvent(api, expectedParams, None, source, context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.of(event, event))) } @@ -500,7 +500,7 @@ class GoogleAnalyticsAdapterSpec extends Specification with DataTables with Vali ).toOpt adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.one(RawEvent(api, expectedParams, None, source, context)))) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/HubSpotAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/HubSpotAdapterSpec.scala index 4c3397d74..1b7f001dd 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/HubSpotAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/HubSpotAdapterSpec.scala @@ -4,8 +4,8 @@ * * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -60,7 +60,7 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidatedMat "subscriptionType": "company.change", "eventId": 16 }""" - adapterWithDefaultSchemas.payloadBodyToEvents(bodyStr) must beRight(List(expected)) + adapterWithDefaultSchemas.payloadBodyToEvents(bodyStr, DefaultMaxJsonDepth) must beRight(List(expected)) } def e2 = @@ -71,7 +71,7 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidatedMat """{"something:"some"}""".some, """invalid json: expected : got 'some"}' (line 1, column 14)""" ) |> { (_, input, expected) => - adapterWithDefaultSchemas.payloadBodyToEvents(input) must beLeft(expected) + adapterWithDefaultSchemas.payloadBodyToEvents(input, DefaultMaxJsonDepth) must beLeft(expected) } def e3 = { @@ -99,7 +99,9 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidatedMat Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e4 = { @@ -119,7 +121,7 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidatedMat "no schema associated with the provided type parameter at index 0" ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one(expected) @@ -131,7 +133,7 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidatedMat val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -146,7 +148,7 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidatedMat val payload = CollectorPayload(Shared.api, Nil, None, "stub".some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -171,7 +173,7 @@ class HubSpotAdapterSpec extends Specification with DataTables with ValidatedMat Shared.context ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/IgluAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/IgluAdapterSpec.scala index 40347c297..f73111f78 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/IgluAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/IgluAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -99,7 +99,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche |}""".stripMargin.replaceAll("[\n\r]", "") IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -149,7 +149,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche } IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -197,7 +197,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche } IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -229,7 +229,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche |}""".stripMargin.replaceAll("[\n\r]", "") IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -250,7 +250,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche val payload = CollectorPayload(Shared.api, params, None, None, Shared.cfSource, Shared.context) IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.of( @@ -270,7 +270,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche val payload = CollectorPayload(Shared.api, params, None, None, Shared.cfSource, Shared.context) IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.of( @@ -289,7 +289,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche val payload = CollectorPayload(Shared.api, params, None, None, Shared.cfSource, Shared.context) IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList @@ -331,7 +331,9 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) - IgluAdapter.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(NonEmptyList.one(expected))) + IgluAdapter + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(NonEmptyList.one(expected))) } def e9 = { @@ -356,7 +358,9 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche "application/badtype".some, "expected one of application/json, application/json; charset=utf-8, application/x-www-form-urlencoded" ) - IgluAdapter.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(NonEmptyList.one(expected))) + IgluAdapter + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(NonEmptyList.one(expected))) } def e10 = { @@ -377,7 +381,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche ) IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -399,7 +403,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche CollectorPayload(Shared.api, params, None, jsonStr.some, Shared.cljSource, Shared.context) IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -440,7 +444,9 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) - IgluAdapter.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(NonEmptyList.one(expected))) + IgluAdapter + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(NonEmptyList.one(expected))) } def e13 = { @@ -458,7 +464,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche ) IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -486,7 +492,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche ) IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -526,7 +532,9 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) - IgluAdapter.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(NonEmptyList.one(expected))) + IgluAdapter + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(NonEmptyList.one(expected))) } def e16 = { @@ -561,7 +569,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche ) IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(NonEmptyList.of(expected, expected))) } @@ -583,7 +591,7 @@ class IgluAdapterSpec extends Specification with DataTables with ValidatedMatche ) IgluAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala index 7fcbc2bcb..993c07507 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailchimpAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -150,7 +150,7 @@ class MailchimpAdapterSpec extends Specification with DataTables with ValidatedM |}""".stripMargin.replaceAll("[\n\r]", "") adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -194,7 +194,7 @@ class MailchimpAdapterSpec extends Specification with DataTables with ValidatedM |}""".stripMargin.replaceAll("[\n\r]", "") adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -230,7 +230,7 @@ class MailchimpAdapterSpec extends Specification with DataTables with ValidatedM val expectedJson = "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"" + expected + "\",\"data\":{\"type\":\"" + schema + "\"}}}" adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -269,7 +269,7 @@ class MailchimpAdapterSpec extends Specification with DataTables with ValidatedM Shared.context ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beInvalid(NonEmptyList.one(expected))) .unsafeRunSync() } @@ -314,7 +314,7 @@ class MailchimpAdapterSpec extends Specification with DataTables with ValidatedM |}""".stripMargin.replaceAll("[\n\r]", "") adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -340,7 +340,7 @@ class MailchimpAdapterSpec extends Specification with DataTables with ValidatedM val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -355,7 +355,7 @@ class MailchimpAdapterSpec extends Specification with DataTables with ValidatedM val payload = CollectorPayload(Shared.api, Nil, None, "stub".some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -379,7 +379,7 @@ class MailchimpAdapterSpec extends Specification with DataTables with ValidatedM Shared.context ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -404,7 +404,7 @@ class MailchimpAdapterSpec extends Specification with DataTables with ValidatedM Shared.context ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala index 937551956..a23b4bf3c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MailgunAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -144,7 +144,9 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e2 = { @@ -211,7 +213,9 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e3 = { @@ -243,7 +247,9 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e4 = { @@ -275,7 +281,9 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e5 = { @@ -360,14 +368,16 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e6 = { val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -383,7 +393,7 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -403,7 +413,7 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat val payload = CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -432,7 +442,9 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat FailureDetails.AdapterFailure .InputData("body", None, "empty body: no events to process") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e10 = { @@ -452,7 +464,9 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat "no `event` parameter provided: cannot determine event type" ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e11 = { @@ -473,7 +487,9 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat "no schema associated with the provided type parameter" ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e12 = { @@ -492,7 +508,9 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat FailureDetails.AdapterFailure .InputData("timestamp", None, "missing 'timestamp'") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e13 = { @@ -509,7 +527,9 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat val expected = NonEmptyList.one( FailureDetails.AdapterFailure.InputData("token", None, "missing 'token'") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e14 = { @@ -528,6 +548,8 @@ class MailgunAdapterSpec extends Specification with DataTables with ValidatedMat FailureDetails.AdapterFailure .InputData("signature", None, "missing 'signature'") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MandrillAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MandrillAdapterSpec.scala index 736e5caaf..0666e9c1d 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MandrillAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MandrillAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -57,7 +57,7 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidatedMa def e1 = { val bodyStr = "mandrill_events=%5B%7B%22event%22%3A%20%22subscribe%22%7D%5D" val expected = List(json"""{"event": "subscribe"}""") - adapterWithDefaultSchemas.payloadBodyToEvents(bodyStr) must beRight(expected) + adapterWithDefaultSchemas.payloadBodyToEvents(bodyStr, SpecHelpers.DefaultMaxJsonDepth) must beRight(expected) } def e2 = @@ -80,7 +80,7 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidatedMa "events_mandrill=something".some, "no `mandrill_events` parameter provided" ) |> { (_, str, expected) => - adapterWithDefaultSchemas.payloadBodyToEvents(str) must beLeft(expected) + adapterWithDefaultSchemas.payloadBodyToEvents(str, SpecHelpers.DefaultMaxJsonDepth) must beLeft(expected) } def e3 = { @@ -90,7 +90,7 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidatedMa """[{"event":"click}]""".some, "invalid json: exhausted input" ) - adapterWithDefaultSchemas.payloadBodyToEvents(bodyStr) must beLeft(expected) + adapterWithDefaultSchemas.payloadBodyToEvents(bodyStr, SpecHelpers.DefaultMaxJsonDepth) must beLeft(expected) } def e4 = { // Spec for nine seperate events being passed and returned. @@ -226,7 +226,9 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e5 = { // Spec for nine seperate events where two have incorrect event names and one does not have event as a parameter @@ -257,14 +259,16 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidatedMa "cannot determine event type: type parameter not provided at index 2" ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e6 = { val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -280,7 +284,7 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidatedMa val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -300,7 +304,7 @@ class MandrillAdapterSpec extends Specification with DataTables with ValidatedMa val payload = CollectorPayload(Shared.api, Nil, ct, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MarketoAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MarketoAdapterSpec.scala index 788c804cc..daec58a20 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MarketoAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/MarketoAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -72,14 +72,16 @@ class MarketoAdapterSpec extends Specification with DataTables with ValidatedMat Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e2 = { val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala index ebd9cb0f2..b0285d094 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/OlarkAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -142,7 +142,9 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidatedMatch Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e2 = { @@ -206,14 +208,16 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidatedMatch Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e3 = { val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -230,7 +234,7 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidatedMatch val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -251,7 +255,7 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidatedMatch val payload = CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -280,7 +284,9 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidatedMatch FailureDetails.AdapterFailure .InputData("body", None, "empty body: no events to process") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e7 = { @@ -297,7 +303,9 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidatedMatch val expected = NonEmptyList.one( FailureDetails.AdapterFailure.InputData("data", None, "missing 'data' field") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e8 = { @@ -312,7 +320,7 @@ class OlarkAdapterSpec extends Specification with DataTables with ValidatedMatch Shared.context ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beInvalid.like { case nel => nel.size must_== 1 diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PagerdutyAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PagerdutyAdapterSpec.scala index 67e28438f..fdbcd4870 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PagerdutyAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PagerdutyAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -90,7 +90,7 @@ class PagerdutyAdapterSpec extends Specification with DataTables with ValidatedM } } }""") - adapterWithDefaultSchemas.payloadBodyToEvents(bodyStr) must beRight(expected) + adapterWithDefaultSchemas.payloadBodyToEvents(bodyStr, DefaultMaxJsonDepth) must beRight(expected) } def e5 = @@ -107,7 +107,7 @@ class PagerdutyAdapterSpec extends Specification with DataTables with ValidatedM """{"somekey":"key"}""".some, "field `messages` is not an array" ) |> { (_, input, expected) => - adapterWithDefaultSchemas.payloadBodyToEvents(input) must beLeft(expected) + adapterWithDefaultSchemas.payloadBodyToEvents(input, DefaultMaxJsonDepth) must beLeft(expected) } def e6 = { @@ -135,7 +135,9 @@ class PagerdutyAdapterSpec extends Specification with DataTables with ValidatedM Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e7 = { @@ -155,7 +157,7 @@ class PagerdutyAdapterSpec extends Specification with DataTables with ValidatedM "no schema associated with the provided type parameter at index 0" ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one(expected) @@ -167,7 +169,7 @@ class PagerdutyAdapterSpec extends Specification with DataTables with ValidatedM val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -182,7 +184,7 @@ class PagerdutyAdapterSpec extends Specification with DataTables with ValidatedM val payload = CollectorPayload(Shared.api, Nil, None, "stub".some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -207,7 +209,7 @@ class PagerdutyAdapterSpec extends Specification with DataTables with ValidatedM Shared.context ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PingdomAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PingdomAdapterSpec.scala index 0c91adc7f..150e187da 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PingdomAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/PingdomAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -93,7 +93,7 @@ class PingdomAdapterSpec extends Specification with DataTables with ValidatedMat Shared.context ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one(expected) @@ -110,7 +110,7 @@ class PingdomAdapterSpec extends Specification with DataTables with ValidatedMat "empty querystring: no events to process" ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one(expected) @@ -129,7 +129,7 @@ class PingdomAdapterSpec extends Specification with DataTables with ValidatedMat "no `message` parameter provided" ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one(expected) diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/RemoteAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/RemoteAdapterSpec.scala index c48eb8c57..4d940d7e1 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/RemoteAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/RemoteAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -176,7 +176,7 @@ class RemoteAdapterSpec extends Specification with ValidatedMatchers with CatsEf SpecHelpers.httpClient.use { http => adapter(http) - .toRawEvents(payload) + .toRawEvents(payload, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid(expected)) } } @@ -193,7 +193,7 @@ class RemoteAdapterSpec extends Specification with ValidatedMatchers with CatsEf ) SpecHelpers.httpClient.use { http => adapter(http) - .toRawEvents(emptyListPayload) + .toRawEvents(emptyListPayload, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beInvalid(expected)) } } @@ -210,7 +210,7 @@ class RemoteAdapterSpec extends Specification with ValidatedMatchers with CatsEf ) SpecHelpers.httpClient.use { http => adapter(http) - .toRawEvents(bodylessPayload) + .toRawEvents(bodylessPayload, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beInvalid(expected)) } } @@ -225,7 +225,7 @@ class RemoteAdapterSpec extends Specification with ValidatedMatchers with CatsEf """[REMOTE_ADAPTER] invalid json: expected " got 'invali...' (line 1, column 2)""" ) SpecHelpers.httpClient.use { http => - IO.pure(adapter(http).processResponse(bodylessPayload, invalidJsonResponse) must beLeft(expected)) + IO.pure(adapter(http).processResponse(bodylessPayload, invalidJsonResponse, DefaultMaxJsonDepth) must beLeft(expected)) } } @@ -239,7 +239,7 @@ class RemoteAdapterSpec extends Specification with ValidatedMatchers with CatsEf "[REMOTE_ADAPTER] could not be decoded as a list of json objects: Got value '\"response\"' with wrong type, expecting array: DownField(events)" ) SpecHelpers.httpClient.use { http => - IO.pure(adapter(http).processResponse(bodylessPayload, unexpectedJsonResponse) must beLeft(expected)) + IO.pure(adapter(http).processResponse(bodylessPayload, unexpectedJsonResponse, DefaultMaxJsonDepth) must beLeft(expected)) } } @@ -253,7 +253,7 @@ class RemoteAdapterSpec extends Specification with ValidatedMatchers with CatsEf "[REMOTE_ADAPTER] empty list of events" ) SpecHelpers.httpClient.use { http => - IO.pure(adapter(http).processResponse(bodylessPayload, emptyJsonResponse) must beLeft(expected)) + IO.pure(adapter(http).processResponse(bodylessPayload, emptyJsonResponse, DefaultMaxJsonDepth) must beLeft(expected)) } } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala index 712540b31..1fa2a033d 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/SendgridAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -223,7 +223,8 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats Shared.cljSource, Shared.context ) - val actual = adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + val actual = + adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) "return the correct number of events" in { actual.map { output => @@ -272,7 +273,9 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats "reject empty bodies" in { val invalidpayload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) - adapterWithDefaultSchemas.toRawEvents(invalidpayload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid) + adapterWithDefaultSchemas + .toRawEvents(invalidpayload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid) } "reject empty content type" in { @@ -285,7 +288,9 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats Shared.cljSource, Shared.context ) - adapterWithDefaultSchemas.toRawEvents(invalidpayload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid) + adapterWithDefaultSchemas + .toRawEvents(invalidpayload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid) } "reject unexpected content type" in { @@ -298,7 +303,9 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats Shared.cljSource, Shared.context ) - adapterWithDefaultSchemas.toRawEvents(invalidpayload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid) + adapterWithDefaultSchemas + .toRawEvents(invalidpayload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid) } "accept content types with explicit charsets" in { @@ -311,7 +318,9 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats Shared.cljSource, Shared.context ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid) } "reject unsupported event types" in { @@ -340,7 +349,9 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats Shared.context ) - adapterWithDefaultSchemas.toRawEvents(invalidpayload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid) + adapterWithDefaultSchemas + .toRawEvents(invalidpayload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid) } "reject invalid/unparsable json" in { @@ -356,7 +367,8 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats Shared.context ), SpecHelpers.client, - SpecHelpers.registryLookup + SpecHelpers.registryLookup, + SpecHelpers.DefaultMaxJsonDepth ) .map(_ must beInvalid) } @@ -374,7 +386,8 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats Shared.context ), SpecHelpers.client, - SpecHelpers.registryLookup + SpecHelpers.registryLookup, + SpecHelpers.DefaultMaxJsonDepth ) .map(_ must beInvalid) } @@ -412,7 +425,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats Shared.context ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -460,7 +473,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.sendgrid/processed/jsonschema/3-0-0","data":{"timestamp":"2015-11-03T11:20:15.000Z","email":"example@test.com","marketing_campaign_name":"campaign name","sg_event_id":"sZROwMGMagFgnOEmSdvhig==","smtp-id":"\u003c14c5d75ce93.dfd.64b469@ismtpd-555\u003e","marketing_campaign_version":"B","marketing_campaign_id":12345,"marketing_campaign_split_id":13471,"category":"cat facts","sg_message_id":"14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}}}""" adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -525,7 +538,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beValid.like { case nel: NonEmptyList[RawEvent] => nel.toList must have size 1 @@ -589,7 +602,7 @@ class SendgridAdapterSpec extends Specification with ValidatedMatchers with Cats """{"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.sendgrid/deferred/jsonschema/3-0-0","data":{"timestamp":"2015-11-03T11:20:15.000Z","email":"example@test.com","marketing_campaign_name":"campaign name","sg_event_id":"jWmZXTZbtHTV2-S47asrww==","smtp-id":"<14c5d75ce93.dfd.64b469@ismtpd-555>","marketing_campaign_version":"B","response":"400 try again later","marketing_campaign_id":12345,"marketing_campaign_split_id":13471,"category":"cat facts","attempt":"5","sg_message_id":"14c5d75ce93.dfd.64b469.filter0001.16648.5515E0B88.0"}}}""" adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.of( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/StatusGatorAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/StatusGatorAdapterSpec.scala index 9d2c9a391..d65d624aa 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/StatusGatorAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/StatusGatorAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -88,14 +88,16 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validate Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e2 = { val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -112,7 +114,7 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validate val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -133,7 +135,7 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validate val payload = CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -162,7 +164,9 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validate FailureDetails.AdapterFailure .InputData("body", None, "empty body: no events to process") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e6 = { @@ -183,6 +187,8 @@ class StatusGatorAdapterSpec extends Specification with DataTables with Validate "could not parse body: Illegal character in query at index 18: http://localhost/?{service_name=CloudFlare&favicon_url=https%3A%2F%2Fdwxjd9cd6rwno.cloudfront.net%2Ffavicons%2Fcloudflare.ico&status_page_url=https%3A%2F%2Fwww.cloudflarestatus.com%2F&home_page_url=http%3A%2F%2Fwww.cloudflare.com¤t_status=up&last_status=warn&occurred_at=2016-05-19T09%3A26%3A31%2B00%3A00" ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UnbounceAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UnbounceAdapterSpec.scala index 290c72dbc..2d03efd70 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UnbounceAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UnbounceAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -117,7 +117,9 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e2 = { @@ -125,7 +127,7 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa val payload = CollectorPayload(Shared.api, params, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -142,7 +144,7 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa val payload = CollectorPayload(Shared.api, Nil, None, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -163,7 +165,7 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa val payload = CollectorPayload(Shared.api, Nil, ct.some, body.some, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -193,7 +195,9 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa FailureDetails.AdapterFailure .InputData("body", None, "empty body: no events to process") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e6 = { @@ -212,7 +216,9 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa FailureDetails.AdapterFailure .InputData("data.json", None, "missing 'data.json' field in body") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e7 = { @@ -231,7 +237,9 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa FailureDetails.AdapterFailure .InputData("data.json", None, "empty 'data.json' field in body") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e8 = { @@ -253,7 +261,9 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa """invalid json: expected " got '{"emai...' (line 1, column 2)""" ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e9 = { @@ -273,7 +283,9 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa FailureDetails.AdapterFailure .InputData("page_id", None, "missing 'page_id' field in body") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e10 = { @@ -292,7 +304,9 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa FailureDetails.AdapterFailure .InputData("page_name", None, "missing 'page_name' field in body") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e11 = { @@ -312,7 +326,9 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa FailureDetails.AdapterFailure .InputData("variant", None, "missing 'variant' field in body") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } def e12 = { @@ -331,6 +347,8 @@ class UnbounceAdapterSpec extends Specification with DataTables with ValidatedMa FailureDetails.AdapterFailure .InputData("page_url", None, "missing 'page_url' field in body") ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid(expected)) } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UrbanAirshipAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UrbanAirshipAdapterSpec.scala index 53ac260c2..a28eb8a9e 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UrbanAirshipAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/UrbanAirshipAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -82,7 +82,8 @@ class UrbanAirshipAdapterSpec extends Specification with ValidatedMatchers with Shared.cljSource, Shared.context ) - val actual = adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + val actual = + adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) val expectedUnstructEventJson = json"""{ "schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0", @@ -134,13 +135,17 @@ class UrbanAirshipAdapterSpec extends Specification with ValidatedMatchers with Shared.cljSource, Shared.context ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid) } "reject unparsable json" in { val payload = CollectorPayload(Shared.api, Nil, None, """{ """.some, Shared.cljSource, Shared.context) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid) } "reject badly formatted json" in { @@ -153,7 +158,9 @@ class UrbanAirshipAdapterSpec extends Specification with ValidatedMatchers with Shared.cljSource, Shared.context ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beInvalid) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beInvalid) } "reject content types" in { @@ -166,7 +173,7 @@ class UrbanAirshipAdapterSpec extends Specification with ValidatedMatchers with Shared.context ) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/VeroAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/VeroAdapterSpec.scala index 6bd11df80..9adeed62d 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/VeroAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/VeroAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -85,7 +85,9 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e2 = { @@ -113,7 +115,9 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e3 = { @@ -141,7 +145,9 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e4 = { @@ -169,7 +175,9 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e5 = { @@ -197,7 +205,9 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e6 = { @@ -225,7 +235,9 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e7 = { @@ -253,7 +265,9 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e8 = { @@ -281,7 +295,9 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche Shared.context ) ) - adapterWithDefaultSchemas.toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup).map(_ must beValid(expected)) + adapterWithDefaultSchemas + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .map(_ must beValid(expected)) } def e9 = @@ -306,7 +322,7 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche val expectedJson = "{\"schema\":\"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0\",\"data\":{\"schema\":\"" + expected + "\",\"data\":{}}}" adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -327,7 +343,7 @@ class VeroAdapterSpec extends Specification with DataTables with ValidatedMatche val payload = CollectorPayload(Shared.api, Nil, ContentType.some, None, Shared.cljSource, Shared.context) adapterWithDefaultSchemas - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/snowplow/SnowplowAdapterSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/snowplow/SnowplowAdapterSpec.scala index 58ce26dfb..cb5b0e5c0 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/snowplow/SnowplowAdapterSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/adapters/registry/snowplow/SnowplowAdapterSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -96,7 +96,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) Tp1Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -115,7 +115,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa def e2 = { val payload = CollectorPayload(Snowplow.Tp1, Nil, None, None, Shared.source, Shared.context) Tp1Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -139,7 +139,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) Tp2Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -168,7 +168,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) Tp2Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -207,7 +207,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa ) Tp2Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.of( @@ -231,7 +231,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) Tp2Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -302,7 +302,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) Tp2Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beInvalid(expected)) .unsafeRunSync() } @@ -317,7 +317,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) Tp2Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -339,7 +339,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) Tp2Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -477,7 +477,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa ) Tp2Adapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map(_ must beInvalid(expected)) .unsafeRunSync() } @@ -495,7 +495,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -531,7 +531,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -568,7 +568,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -604,7 +604,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -642,7 +642,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beValid( NonEmptyList.one( @@ -668,7 +668,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa def e16 = { val payload = CollectorPayload(Snowplow.Tp2, Nil, None, None, Shared.source, Shared.context) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -693,7 +693,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -721,7 +721,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -749,7 +749,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( @@ -773,7 +773,7 @@ class SnowplowAdapterSpec extends Specification with DataTables with ValidatedMa Shared.context ) RedirectAdapter - .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup) + .toRawEvents(payload, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .map( _ must beInvalid( NonEmptyList.one( diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/AtomicFieldsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/AtomicFieldsSpec.scala index 8117eaa12..47c1cf8db 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/AtomicFieldsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/AtomicFieldsSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/CachingEvaluatorSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/CachingEvaluatorSpec.scala index 34df2a432..8c61e9d22 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/CachingEvaluatorSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/CachingEvaluatorSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala index fbc74f052..cb595c5e7 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentManagerSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -92,7 +92,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value map { case Ior.Left(_: BadRow.SchemaViolations) => ok @@ -130,7 +131,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value map { case Ior.Left(_: BadRow.SchemaViolations) => ok @@ -170,7 +172,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -238,7 +241,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value map { case Ior.Left( @@ -307,7 +311,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value map { case Ior.Left( @@ -372,7 +377,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(_) => ok @@ -442,7 +448,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(_) => ok @@ -512,7 +519,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(_) => ok @@ -582,7 +590,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Left(_) => ok @@ -652,7 +661,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Left(_) => ok @@ -728,7 +738,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Left(_) => ok @@ -760,7 +771,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(_) => ok @@ -794,7 +806,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(_) => ok @@ -837,7 +850,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(enriched) => enriched.se_value.toString must_== expected @@ -866,7 +880,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(enriched) => @@ -894,7 +909,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(enriched) => enriched.useragent must_== "header-useragent" @@ -921,7 +937,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(enriched) => enriched.useragent must_== ua @@ -949,7 +966,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(enriched) => @@ -997,7 +1015,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(enriched) => enriched.app_id must_== "moo" @@ -1049,7 +1068,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Right(enriched) => @@ -1206,7 +1226,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { @@ -1303,7 +1324,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete = true + emitIncomplete = true, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { @@ -1401,7 +1423,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete = true + emitIncomplete = true, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Both(_: BadRow.SchemaViolations, enriched) @@ -1503,7 +1526,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete = true + emitIncomplete = true, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Both(_: BadRow.SchemaViolations, enriched) @@ -1575,7 +1599,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete = true + emitIncomplete = true, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Both(_: BadRow.EnrichmentFailures, enriched) @@ -1658,7 +1683,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete = true + emitIncomplete = true, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Both(_: BadRow.SchemaViolations, enriched) if expectedDerivedContexts(enriched) => ok @@ -1750,7 +1776,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete = true + emitIncomplete = true, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { case Ior.Both(_: BadRow.SchemaViolations, enriched) @@ -1818,7 +1845,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete = true + emitIncomplete = true, + SpecHelpers.DefaultMaxJsonDepth ) def expectedDerivedContexts(enriched: EnrichedEvent): Boolean = SpecHelpers.listContextsSchemas(enriched.derived_contexts).count(_ == Failure.failureSchemaKey) == 2 @@ -2320,7 +2348,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -2351,7 +2380,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -2401,7 +2431,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -2438,7 +2469,8 @@ class EnrichmentManagerSpec extends Specification with EitherMatchers with CatsE IO.unit, SpecHelpers.registryLookup, atomicFieldLimits, - emitIncomplete = true + emitIncomplete = true, + SpecHelpers.DefaultMaxJsonDepth ) enriched.value.map { diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentRegistrySpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentRegistrySpec.scala index d997c5668..1d1feccd5 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentRegistrySpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/EnrichmentRegistrySpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/FailureSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/FailureSpec.scala index d8dee9940..4139acd35 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/FailureSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/FailureSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentSpec.scala index 0d9a47908..b045139f9 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/SchemaEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/clientEnrichmentSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/clientEnrichmentSpecs.scala index c393cc5d7..2abfbd024 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/clientEnrichmentSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/clientEnrichmentSpecs.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/eventEnrichmentSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/eventEnrichmentSpecs.scala index 1b8ba8837..4331d2caa 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/eventEnrichmentSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/eventEnrichmentSpecs.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/miscEnrichmentSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/miscEnrichmentSpecs.scala index 37e751fc9..368921545 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/miscEnrichmentSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/miscEnrichmentSpecs.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/AnonIpEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/AnonIpEnrichmentSpec.scala index e60d05007..68d85d802 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/AnonIpEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/AnonIpEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CampaignAttributionEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CampaignAttributionEnrichmentSpec.scala index 3f4fd044d..b78616e10 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CampaignAttributionEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CampaignAttributionEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CookieExtractorEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CookieExtractorEnrichmentSpec.scala index ced51ca32..078fe987f 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CookieExtractorEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CookieExtractorEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * | * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CrossNavigationEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CrossNavigationEnrichmentSpec.scala index 387f868f1..e68b06b8c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CrossNavigationEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CrossNavigationEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala index 1064f7b19..c05f2a248 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/CurrencyConversionEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala index 01c97895a..7fc76d53e 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EnrichmentConfigsSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala index b3bdef2f8..fbb0a8843 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/EventFingerprintEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/HttpHeaderExtractorEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/HttpHeaderExtractorEnrichmentSpec.scala index 93826131b..c73be6851 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/HttpHeaderExtractorEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/HttpHeaderExtractorEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IabEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IabEnrichmentSpec.scala index 46f1949d5..6bc29ca3c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IabEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IabEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IpLookupsEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IpLookupsEnrichmentSpec.scala index c170e8e2a..333504640 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IpLookupsEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/IpLookupsEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala index d596b8c86..5d72c4307 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/JavascriptScriptEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -21,6 +21,7 @@ import com.snowplowanalytics.iglu.core.{SchemaKey, SchemaVer, SelfDescribingData import com.snowplowanalytics.snowplow.badrows.FailureDetails import com.snowplowanalytics.snowplow.enrich.common.outputs.EnrichedEvent +import com.snowplowanalytics.snowplow.enrich.common.SpecHelpers class JavascriptScriptEnrichmentSpec extends Specification { def is = s2""" @@ -36,13 +37,14 @@ class JavascriptScriptEnrichmentSpec extends Specification { Javascript enrichment should be able to proceed with return null $e10 Javascript enrichment should be able to update the fields without return statement $e11 Javascript enrichment should be able to utilize the passed parameters $e12 + Javascript enrichment should be able to utilize the headers $e13 """ val schemaKey = SchemaKey("com.snowplowanalytics.snowplow", "javascript_script_config", "jsonschema", SchemaVer.Full(1, 0, 0)) def e1 = - JavascriptScriptEnrichment(schemaKey, "[").process(buildEnriched()) must beLeft( + JavascriptScriptEnrichment(schemaKey, "[").process(buildEnriched(), List.empty, SpecHelpers.DefaultMaxJsonDepth) must beLeft( failureContains(_: FailureDetails.EnrichmentFailure, "Error compiling") ) @@ -51,7 +53,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { function process(event) { return { foo: "bar" } }""" - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beLeft( + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched(), List.empty, SpecHelpers.DefaultMaxJsonDepth) must beLeft( failureContains(_: FailureDetails.EnrichmentFailure, "not read as an array") ) } @@ -61,7 +63,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { function process(event) { return [ { foo: "bar" } ] }""" - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beLeft( + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched(), List.empty, SpecHelpers.DefaultMaxJsonDepth) must beLeft( failureContains(_: FailureDetails.EnrichmentFailure, "not self-desribing") ) } @@ -74,10 +76,11 @@ class JavascriptScriptEnrichmentSpec extends Specification { data: { appId: event.getApp_id() } } ]; }""" - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched(appId)) must beRight.like { - case List(sdj) if sdj.data.noSpaces.contains(appId) => true - case _ => false - } + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched(appId), List.empty, SpecHelpers.DefaultMaxJsonDepth) must beRight + .like { + case List(sdj) if sdj.data.noSpaces.contains(appId) => true + case _ => false + } } def e5 = { @@ -91,7 +94,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { data: { foo: "bar" } } ]; }""" - JavascriptScriptEnrichment(schemaKey, function).process(enriched) + JavascriptScriptEnrichment(schemaKey, function).process(enriched, List.empty, SpecHelpers.DefaultMaxJsonDepth) enriched.app_id must beEqualTo(newAppId) } @@ -100,7 +103,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { function process(event) { throw "Error" }""" - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beLeft( + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched(), List.empty, SpecHelpers.DefaultMaxJsonDepth) must beLeft( failureContains(_: FailureDetails.EnrichmentFailure, "Error during execution") ) } @@ -110,7 +113,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { function process(event) { return [ ]; }""" - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beRight + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched(), List.empty, SpecHelpers.DefaultMaxJsonDepth) must beRight } def e8 = { @@ -133,10 +136,11 @@ class JavascriptScriptEnrichmentSpec extends Specification { SchemaKey("com.acme", "bar", "jsonschema", SchemaVer.Full(1, 0, 0)), json"""{"hello":"world"}""" ) - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beRight.like { - case List(c1, c2) if c1 == context1 && c2 == context2 => true - case _ => false - } + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched(), List.empty, SpecHelpers.DefaultMaxJsonDepth) must beRight + .like { + case List(c1, c2) if c1 == context1 && c2 == context2 => true + case _ => false + } } def e9 = { @@ -145,7 +149,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { var a = 42 // no-op }""" - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beRight(Nil) + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched(), List.empty, SpecHelpers.DefaultMaxJsonDepth) must beRight(Nil) } def e10 = { @@ -154,7 +158,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { return null }""" - JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched()) must beRight(Nil) + JavascriptScriptEnrichment(schemaKey, function).process(buildEnriched(), List.empty, SpecHelpers.DefaultMaxJsonDepth) must beRight(Nil) } def e11 = { @@ -165,7 +169,7 @@ class JavascriptScriptEnrichmentSpec extends Specification { function process(event) { event.setApp_id("$newAppId") }""" - JavascriptScriptEnrichment(schemaKey, function).process(enriched) + JavascriptScriptEnrichment(schemaKey, function).process(enriched, List.empty, SpecHelpers.DefaultMaxJsonDepth) enriched.app_id must beEqualTo(newAppId) } @@ -178,7 +182,28 @@ class JavascriptScriptEnrichmentSpec extends Specification { function process(event, params) { event.setApp_id(params.nested.foo) }""" - JavascriptScriptEnrichment(schemaKey, function, params).process(enriched) + JavascriptScriptEnrichment(schemaKey, function, params).process(enriched, List.empty, SpecHelpers.DefaultMaxJsonDepth) + enriched.app_id must beEqualTo("newId") + } + + def e13 = { + val appId = "greatApp" + val enriched = buildEnriched(appId) + val function = + s""" + function process(event, params, headers) { + for (header of headers) { + const jwt = header.match(/X-JWT:(.+)/i) + if (jwt) { + event.setApp_id(jwt[1].trim()) + } + } + }""" + + JavascriptScriptEnrichment(schemaKey, function).process(enriched, List.empty, SpecHelpers.DefaultMaxJsonDepth) + enriched.app_id must beEqualTo("greatApp") + + JavascriptScriptEnrichment(schemaKey, function).process(enriched, List("x-jwt: newId"), SpecHelpers.DefaultMaxJsonDepth) enriched.app_id must beEqualTo("newId") } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/RefererParserEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/RefererParserEnrichmentSpec.scala index 38752f853..16495ca63 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/RefererParserEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/RefererParserEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala index 6849ae4b7..a3bd187c7 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UaParserEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UserAgentUtilsEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UserAgentUtilsEnrichmentSpec.scala index ccdeb361c..831a768ba 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UserAgentUtilsEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/UserAgentUtilsEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala index 74482f90b..0e3485914 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/WeatherEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala index d1991a711..8d3bd4430 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/YauaaEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentIntegrationTest.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentIntegrationTest.scala index 7f462abc9..5e16df235 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentIntegrationTest.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentIntegrationTest.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala index 1f686ca04..98462b962 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ApiRequestEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala index df6586d1a..74aa71241 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpApiSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpClientSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpClientSpec.scala index 5716e1671..0f367d5e7 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpClientSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/HttpClientSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala index 3f88c50d8..8844173ae 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/InputSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/OutputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/OutputSpec.scala index 5b06ecb5e..978ed7e9c 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/OutputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/OutputSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ValidatorSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ValidatorSpec.scala index 41d3b6834..6d586e418 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ValidatorSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/apirequest/ValidatorSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala index 25148a1cc..099a257e0 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/pii/PiiPseudonymizerEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -169,7 +169,7 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidatedMatcher ) val reg = Registry.Embedded(regConf, path = "/iglu-schemas") for { - client <- IgluCirceClient.fromResolver[IO](Resolver(List(reg), None), cacheSize = 0) + client <- IgluCirceClient.fromResolver[IO](Resolver[IO](List(reg), None), cacheSize = 0, maxJsonDepth = 40) result <- EtlPipeline .processEvents[IO]( new AdapterRegistry[IO](Map.empty[(String, String), RemoteAdapter[IO]], adaptersSchemas), @@ -182,7 +182,8 @@ class PiiPseudonymizerEnrichmentSpec extends Specification with ValidatedMatcher IO.unit, SpecHelpers.registryLookup, AtomicFields.from(Map.empty), - emitIncomplete + emitIncomplete, + SpecHelpers.DefaultMaxJsonDepth ) } yield result.map(_.toEither) } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala index e1131e2b3..8f938df13 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/InputSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/OutputSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/OutputSpec.scala index d907d5190..8626377d0 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/OutputSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/OutputSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala index f499c7a52..32c7ce82a 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentIntegrationTest.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala index 604c02b94..95b0150fd 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/registry/sqlquery/SqlQueryEnrichmentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala index 862805309..555db1456 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ExtractPageUriSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ParseCrossDomainSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ParseCrossDomainSpec.scala index 154bde8d1..19ebbc69d 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ParseCrossDomainSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/enrichments/web/ParseCrossDomainSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CollectorPayloadSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CollectorPayloadSpec.scala index 66cbf7daf..47aa73481 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CollectorPayloadSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/CollectorPayloadSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/IpAddressExtractorSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/IpAddressExtractorSpec.scala index ecf640146..96c792c91 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/IpAddressExtractorSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/IpAddressExtractorSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/LoaderSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/LoaderSpec.scala index 5ff5050ad..3a4b481c5 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/LoaderSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/LoaderSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/ThriftLoaderSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/ThriftLoaderSpec.scala index cc4f5b112..f36d5f148 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/ThriftLoaderSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/loaders/ThriftLoaderSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/outputs/EnrichedEventSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/outputs/EnrichedEventSpec.scala index 8ef48f684..46aa6f421 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/outputs/EnrichedEventSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/outputs/EnrichedEventSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/CirceUtilsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/CirceUtilsSpec.scala index def16953b..c79155d4e 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/CirceUtilsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/CirceUtilsSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala index 1f8f492a6..54e53352b 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/IgluUtilsSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -35,6 +35,7 @@ import com.snowplowanalytics.snowplow.enrich.common.adapters.RawEvent import com.snowplowanalytics.snowplow.enrich.common.loaders.CollectorPayload class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect { + import IgluUtilsSpec._ val raw = RawEvent( CollectorPayload.Api("vendor", "version"), @@ -53,20 +54,6 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect val notJson = "foo" val jsonNotJson = notJson.asJson // Just jsonized version of the string val notIglu = """{"foo":"bar"}""" - val unstructSchema = - SchemaKey( - "com.snowplowanalytics.snowplow", - "unstruct_event", - "jsonschema", - SchemaVer.Full(1, 0, 0) - ) - val inputContextsSchema = - SchemaKey( - "com.snowplowanalytics.snowplow", - "contexts", - "jsonschema", - SchemaVer.Full(1, 0, 0) - ) val emailSentSchema = SchemaKey( "com.acme", @@ -142,10 +129,15 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect val noSchema = s"""{"schema":"iglu:com.snowplowanalytics.snowplow/foo/jsonschema/1-0-0", "data": $noSchemaData}""" + val deepJson = s"""{ + "schema": "${emailSentSchema.toSchemaUri}", + "data": {"d1":{"d2":{"d3":{"d4":{"d5":{"d6":6}}}}}} + }""" + "extractAndValidateUnstructEvent" should { "return None if unstruct_event field is empty" >> { IgluUtils - .extractAndValidateUnstructEvent(new EnrichedEvent, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(new EnrichedEvent, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Right(None) => ok @@ -158,7 +150,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setUnstruct_event(notJson) IgluUtils - .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -179,7 +171,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setUnstruct_event(notIglu) IgluUtils - .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -197,7 +189,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setUnstruct_event(noSchema) IgluUtils - .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -219,7 +211,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setUnstruct_event(ue) IgluUtils - .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both(NonEmptyList( @@ -240,7 +232,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setUnstruct_event(ue) IgluUtils - .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both(NonEmptyList(Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), @@ -263,7 +255,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setUnstruct_event(buildUnstruct(invalidEmailSent)) IgluUtils - .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -284,7 +276,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setUnstruct_event(buildUnstruct(noSchema)) IgluUtils - .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -304,7 +296,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setUnstruct_event(buildUnstruct(emailSent1)) IgluUtils - .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Right(Some(IgluUtils.SdjExtractResult(sdj, None))) if sdj.schema == emailSentSchema => ok @@ -326,7 +318,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect val expectedValidationInfo = IgluUtils.ValidationInfo(supersedingExampleSchema100, supersedingExampleSchema101.version) IgluUtils - .extractAndValidateUnstructEvent(input1, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input1, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Right(Some(IgluUtils.SdjExtractResult(sdj, Some(`expectedValidationInfo`)))) @@ -341,7 +333,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect // input2 wouldn't be validated with 1-0-0. It would be validated with 1-0-1 only. IgluUtils - .extractAndValidateUnstructEvent(input2, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateUnstructEvent(input2, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Right(Some(IgluUtils.SdjExtractResult(sdj, Some(`expectedValidationInfo`)))) @@ -354,12 +346,34 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect case other => ko(s"no unstructured event was extracted [$other]") } } + + "return a FailureDetails.SchemaViolation.IgluError containing a ValidationError if the JSON in .data exceeds the max allowed JSON depth" >> { + val igluScalaClient = SpecHelpers.client(5) + val input = new EnrichedEvent + val json = deepJson.toJson + input.setUnstruct_event(buildUnstruct(deepJson)) + + IgluUtils + .extractAndValidateUnstructEvent(input, igluScalaClient, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .value + .map { + case Ior.Both( + NonEmptyList( + Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `unstructFieldName`, `json`, _), + _ + ), + None + ) => + ok + case other => ko(s"[$other] is not expected one") + } + } } "extractAndValidateInputContexts" should { "return Nil if contexts field is empty" >> { IgluUtils - .extractAndValidateInputContexts(new EnrichedEvent, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(new EnrichedEvent, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Right(Nil) => ok @@ -372,7 +386,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(notJson) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -392,7 +406,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(notIglu) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -410,7 +424,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(noSchema) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both(NonEmptyList( @@ -432,7 +446,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(notArrayContexts) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -453,7 +467,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(buildInputContexts(List(invalidEmailSent))) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -474,7 +488,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(buildInputContexts(List(noSchema))) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both( @@ -496,7 +510,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(buildInputContexts(List(invalidEmailSent, noSchema))) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both(NonEmptyList( @@ -526,7 +540,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(buildInputContexts(List(emailSent1, noSchema))) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Both(NonEmptyList( @@ -545,7 +559,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(buildInputContexts(List(emailSent1, emailSent2))) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Right(sdjs) if sdjs.size == 2 && sdjs.forall(i => i.sdj.schema == emailSentSchema && i.validationInfo.isEmpty) => @@ -560,7 +574,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(buildInputContexts(List(clientSession))) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Right(sdjs) if sdjs.size == 1 && sdjs.forall(_.sdj.schema == clientSessionSchema) => @@ -575,7 +589,7 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect input.setContexts(buildInputContexts(List(supersedingExample1, supersedingExample2))) IgluUtils - .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup) + .extractAndValidateInputContexts(input, SpecHelpers.client, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) .value .map { case Ior.Right(sdjs) if sdjs.size == 2 && sdjs.forall(_.sdj.schema == supersedingExampleSchema101) => @@ -584,6 +598,28 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect ko(s"[$other] is not 2 SDJs with expected schema [${supersedingExampleSchema101.toSchemaUri}]") } } + + "return a FailureDetails.SchemaViolation.IgluError containing a ValidationError if .data contains context that exceeds the max allowed JSON depth" >> { + val igluScalaClient = SpecHelpers.client(5) + val input = new EnrichedEvent + val json = s"[$deepJson]".toJson + input.setContexts(buildInputContexts(List(deepJson))) + + IgluUtils + .extractAndValidateInputContexts(input, igluScalaClient, SpecHelpers.registryLookup, SpecHelpers.DefaultMaxJsonDepth) + .value + .map { + case Ior.Both( + NonEmptyList( + Failure.SchemaViolation(FailureDetails.SchemaViolation.IgluError(_, _: ValidationError), `contextsFieldName`, `json`, _), + Nil + ), + Nil + ) => + ok + case other => ko(s"[$other] is not expected one") + } + } } "validateEnrichmentsContexts" should { @@ -696,7 +732,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .extractAndValidateInputJsons( input, SpecHelpers.client, - SpecHelpers.registryLookup + SpecHelpers.registryLookup, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -720,7 +757,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .extractAndValidateInputJsons( input, SpecHelpers.client, - SpecHelpers.registryLookup + SpecHelpers.registryLookup, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -745,7 +783,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .extractAndValidateInputJsons( input, SpecHelpers.client, - SpecHelpers.registryLookup + SpecHelpers.registryLookup, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -770,7 +809,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .extractAndValidateInputJsons( input, SpecHelpers.client, - SpecHelpers.registryLookup + SpecHelpers.registryLookup, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -796,7 +836,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .extractAndValidateInputJsons( input, SpecHelpers.client, - SpecHelpers.registryLookup + SpecHelpers.registryLookup, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -828,7 +869,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .extractAndValidateInputJsons( input, SpecHelpers.client, - SpecHelpers.registryLookup + SpecHelpers.registryLookup, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -865,7 +907,8 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect .extractAndValidateInputJsons( input, SpecHelpers.client, - SpecHelpers.registryLookup + SpecHelpers.registryLookup, + SpecHelpers.DefaultMaxJsonDepth ) .value .map { @@ -883,6 +926,23 @@ class IgluUtilsSpec extends Specification with ValidatedMatchers with CatsEffect } } } +} + +object IgluUtilsSpec { + val unstructSchema = + SchemaKey( + "com.snowplowanalytics.snowplow", + "unstruct_event", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) + val inputContextsSchema = + SchemaKey( + "com.snowplowanalytics.snowplow", + "contexts", + "jsonschema", + SchemaVer.Full(1, 0, 0) + ) def buildUnstruct(sdj: String) = s"""{"schema": "${unstructSchema.toSchemaUri}", "data": $sdj}""" diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala index eab03196d..5acaa5012 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonPathSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonUtilsSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonUtilsSpec.scala index 2b9a4310a..2d9d4a063 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonUtilsSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/JsonUtilsSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -19,12 +19,17 @@ import io.circe.Json import cats.data.NonEmptyList class JsonUtilsSpec extends Specification { + import JsonUtilsSpec._ + def is = s2""" toJson can deal with non-null String $e1 toJson can deal with null String $e2 toJson can deal with booleans $e3 toJson can deal with integers $e4 toJson can deal with dates $e5 + extractJson can parse JSON that doesn't exceed max depth $e6 + extractJson should return Left when given JSON exceeds max depth $e7 + extractJson should return Left when given string is not json $e8 """ def e1 = { @@ -88,4 +93,59 @@ class JsonUtilsSpec extends Specification { exp1 and exp2 } + + def e6 = { + val jsonStr = """ + { + "f1": "v", + "f2": 2, + "f3": true, + "f4": { + "f41": { + "f411": [1, 2, 3], + "f412": "v", + "f413": [ + {"f4131": "v"}, + {"f4132": {"f41321": "v"}} + ] + } + }, + "f5": [1, 2, 3], + "f6": [ + {"f61": "v"}, + {"f62": {"f621": {"f6211": "v"}}} + ] + }""" + JsonUtils.extractJson(jsonStr, 10) must beRight + } + + def e7 = { + val deepJsonObject = createDeepJsonObject(1000000) + val deepJsonArray = createDeepJsonArray(1000000) + + val expectedErrorMessage = "invalid json: maximum allowed JSON depth exceeded" + JsonUtils.extractJson(deepJsonObject, 40) must beLeft(expectedErrorMessage) + JsonUtils.extractJson(deepJsonArray, 40) must beLeft(expectedErrorMessage) + } + + def e8 = + JsonUtils.extractJson("{test:", 40) must beLeft("invalid json: expected \" got 'test:' (line 1, column 2)") +} + +object JsonUtilsSpec { + def createDeepJsonObject(depth: Int): String = + createDeepJson(depth, """{"1":""", "}") + + def createDeepJsonArray(depth: Int): String = + createDeepJson(depth, "[", "]") + + def createDeepJson( + depth: Int, + p: String, + s: String + ): String = { + val prefix = (1 to depth).map(_ => p).mkString + val suffix = (1 to depth).map(_ => s).mkString + s"""$prefix"depth-$depth"$suffix""" + } } diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/MapTransformerSpec.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/MapTransformerSpec.scala index 5e8fc5b25..eb34a1720 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/MapTransformerSpec.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/MapTransformerSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala index 06d8eea5b..a73c70f71 100644 --- a/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala +++ b/modules/common/src/test/scala/com.snowplowanalytics.snowplow.enrich.common/utils/conversionUtilsSpecs.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kafka/src/it/scala/com/snowplowanalytics/snowplow/enrich/kafka/test/EnrichKafkaSpec.scala b/modules/kafka/src/it/scala/com/snowplowanalytics/snowplow/enrich/kafka/test/EnrichKafkaSpec.scala index ea30c37c6..171985561 100644 --- a/modules/kafka/src/it/scala/com/snowplowanalytics/snowplow/enrich/kafka/test/EnrichKafkaSpec.scala +++ b/modules/kafka/src/it/scala/com/snowplowanalytics/snowplow/enrich/kafka/test/EnrichKafkaSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/AzureAuthenticationCallbackHandler.scala b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/AzureAuthenticationCallbackHandler.scala index c9783f6b7..5a0337a01 100644 --- a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/AzureAuthenticationCallbackHandler.scala +++ b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/AzureAuthenticationCallbackHandler.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Main.scala b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Main.scala index 9ae70af42..e23e11f71 100644 --- a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Main.scala +++ b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Main.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Sink.scala b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Sink.scala index 1b098d521..09ccb5e4b 100644 --- a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Sink.scala +++ b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Sink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Source.scala b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Source.scala index c236402be..753315bc5 100644 --- a/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Source.scala +++ b/modules/kafka/src/main/scala/com.snowplowanalytics.snowplow.enrich.kafka/Source.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kafka/src/test/scala/com/snowplowanalytics/snowplow/enrich/kafka/ConfigSpec.scala b/modules/kafka/src/test/scala/com/snowplowanalytics/snowplow/enrich/kafka/ConfigSpec.scala index c2c947bd6..270c42f3d 100644 --- a/modules/kafka/src/test/scala/com/snowplowanalytics/snowplow/enrich/kafka/ConfigSpec.scala +++ b/modules/kafka/src/test/scala/com/snowplowanalytics/snowplow/enrich/kafka/ConfigSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -169,7 +169,8 @@ class ConfigSpec extends Specification with CatsEffect { ) ), io.License(accept = true), - io.Validation(AtomicFields.from(atomicFieldLimitsDefaults ++ Map("app_id" -> 5, "mkt_clickid" -> 100000))) + io.Validation(AtomicFields.from(atomicFieldLimitsDefaults ++ Map("app_id" -> 5, "mkt_clickid" -> 100000))), + maxJsonDepth = 50 ) ConfigFile.parse[IO](configPath.asRight).value.map(result => result must beRight(expected)) } @@ -255,7 +256,8 @@ class ConfigSpec extends Specification with CatsEffect { SpecHelpers.adaptersSchemas, io.BlobStorageClients(gcs = false, s3 = false, azureStorage = None), io.License(accept = true), - io.Validation(AtomicFields.from(atomicFieldLimitsDefaults)) + io.Validation(AtomicFields.from(atomicFieldLimitsDefaults)), + maxJsonDepth = 40 ) ConfigFile.parse[IO](configPath.asRight).value.map(result => result must beRight(expected)) } diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Containers.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Containers.scala index adfd10428..8d0d605a9 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Containers.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Containers.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -40,7 +40,7 @@ object Containers extends CatsEffect { case class DockerImage(image: String, tag: String) { def toStr = s"$image:$tag" } - val Localstack = DockerImage("localstack/localstack-light", "1.2.0") + val Localstack = DockerImage("localstack/localstack", "3.4.0") val Enrich = DockerImage("snowplow/snowplow-enrich-kinesis", s"${BuildInfo.version}-distroless") val MySQL = DockerImage("mysql", "8.0.31") val HTTP = DockerImage("nginx", "1.23.2") diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/EnrichKinesisSpec.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/EnrichKinesisSpec.scala index eb1ce325e..6b5a4cc55 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/EnrichKinesisSpec.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/EnrichKinesisSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -20,7 +20,6 @@ import cats.effect.kernel.Resource import cats.effect.testing.specs2.CatsResource import org.specs2.mutable.SpecificationLike -import org.specs2.specification.BeforeAll import com.snowplowanalytics.snowplow.enrich.kinesis.enrichments._ @@ -28,7 +27,7 @@ import com.snowplowanalytics.snowplow.enrich.common.fs2.test.CollectorPayloadGen import com.snowplowanalytics.snowplow.enrich.kinesis.Containers.Localstack -class EnrichKinesisSpec extends CatsResource[IO, Localstack] with SpecificationLike with BeforeAll { +class EnrichKinesisSpec extends CatsResource[IO, Localstack] with SpecificationLike { override protected val Timeout = 10.minutes diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/KinesisConfig.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/KinesisConfig.scala index 91d279af9..8ad669a84 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/KinesisConfig.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/KinesisConfig.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/ApiRequest.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/ApiRequest.scala index 82f24de2b..54b5b2df0 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/ApiRequest.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/ApiRequest.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Enrichment.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Enrichment.scala index 9bd826491..8006352f2 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Enrichment.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Enrichment.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Enrichments.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Enrichments.scala index a8472b03e..63024866f 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Enrichments.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Enrichments.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Javascript.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Javascript.scala index 9a1207d43..411e817c7 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Javascript.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Javascript.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/SqlQuery.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/SqlQuery.scala index 91c2bc66a..64f9da439 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/SqlQuery.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/SqlQuery.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Yauaa.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Yauaa.scala index b8a84daf5..e7b465ef8 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Yauaa.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/enrichments/Yauaa.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/utils.scala b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/utils.scala index dd5a19775..4a295071f 100644 --- a/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/utils.scala +++ b/modules/kinesis/src/it/scala/com/snowplowanalytics/snowplow/enrich/kinesis/utils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/DynamoDbConfig.scala b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/DynamoDbConfig.scala index 35d9f6ca2..dc46d0180 100644 --- a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/DynamoDbConfig.scala +++ b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/DynamoDbConfig.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Main.scala b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Main.scala index 159dbb9ca..0a1e8dd31 100644 --- a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Main.scala +++ b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Main.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Sink.scala b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Sink.scala index 6196250a1..c9e586733 100644 --- a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Sink.scala +++ b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Sink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Source.scala b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Source.scala index e3881374c..1ac9bb29a 100644 --- a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Source.scala +++ b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/Source.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/package.scala b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/package.scala index 083b4acb6..5d3be007c 100644 --- a/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/package.scala +++ b/modules/kinesis/src/main/scala/com/snowplowanalytics/snowplow/enrich/kinesis/package.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/kinesis/src/test/scala/com/snowplowanalytics/snowplow/enrich/kinesis/ConfigSpec.scala b/modules/kinesis/src/test/scala/com/snowplowanalytics/snowplow/enrich/kinesis/ConfigSpec.scala index ec6e9e12f..d0a025a62 100644 --- a/modules/kinesis/src/test/scala/com/snowplowanalytics/snowplow/enrich/kinesis/ConfigSpec.scala +++ b/modules/kinesis/src/test/scala/com/snowplowanalytics/snowplow/enrich/kinesis/ConfigSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -144,7 +144,8 @@ class ConfigSpec extends Specification with CatsEffect { adaptersSchemas, io.BlobStorageClients(gcs = false, s3 = true, azureStorage = None), io.License(accept = true), - io.Validation(AtomicFields.from(atomicFieldLimitsDefaults ++ Map("app_id" -> 5, "mkt_clickid" -> 100000))) + io.Validation(AtomicFields.from(atomicFieldLimitsDefaults ++ Map("app_id" -> 5, "mkt_clickid" -> 100000))), + maxJsonDepth = 50 ) ConfigFile.parse[IO](configPath.asRight).value.map(result => result must beRight(expected)) } @@ -226,7 +227,8 @@ class ConfigSpec extends Specification with CatsEffect { adaptersSchemas, io.BlobStorageClients(gcs = false, s3 = true, azureStorage = None), io.License(accept = true), - io.Validation(AtomicFields.from(atomicFieldLimitsDefaults)) + io.Validation(AtomicFields.from(atomicFieldLimitsDefaults)), + maxJsonDepth = 40 ) ConfigFile.parse[IO](configPath.asRight).value.map(result => result must beRight(expected)) } diff --git a/modules/kinesis/src/test/scala/com/snowplowanalytics/snowplow/enrich/kinesis/SinkSpec.scala b/modules/kinesis/src/test/scala/com/snowplowanalytics/snowplow/enrich/kinesis/SinkSpec.scala index 249fba9da..a997083c1 100644 --- a/modules/kinesis/src/test/scala/com/snowplowanalytics/snowplow/enrich/kinesis/SinkSpec.scala +++ b/modules/kinesis/src/test/scala/com/snowplowanalytics/snowplow/enrich/kinesis/SinkSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/Containers.scala b/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/Containers.scala index 487589a84..6c9421407 100644 --- a/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/Containers.scala +++ b/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/Containers.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/EnrichNsqSpec.scala b/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/EnrichNsqSpec.scala index 74efd9ccc..bc7573f4f 100644 --- a/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/EnrichNsqSpec.scala +++ b/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/EnrichNsqSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/Utils.scala b/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/Utils.scala index 82db69d82..7be93d8bc 100644 --- a/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/Utils.scala +++ b/modules/nsq/src/it/scala/com/snowplowanalytics/snowplow/enrich/nsq/test/Utils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Main.scala b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Main.scala index a479c458e..ae3c982f6 100644 --- a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Main.scala +++ b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Main.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Record.scala b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Record.scala index 35a8a1214..f96714f6c 100644 --- a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Record.scala +++ b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Record.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Sink.scala b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Sink.scala index 7016a2b76..c859e80d3 100644 --- a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Sink.scala +++ b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Sink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Source.scala b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Source.scala index 3136cc336..009fd0623 100644 --- a/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Source.scala +++ b/modules/nsq/src/main/scala/com/snowplowanalytics/snowplow/enrich/nsq/Source.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/nsq/src/test/scala/com/snowplowanalytics/snowplow/enrich/nsq/ConfigSpec.scala b/modules/nsq/src/test/scala/com/snowplowanalytics/snowplow/enrich/nsq/ConfigSpec.scala index 442e15205..2a36ebdb7 100644 --- a/modules/nsq/src/test/scala/com/snowplowanalytics/snowplow/enrich/nsq/ConfigSpec.scala +++ b/modules/nsq/src/test/scala/com/snowplowanalytics/snowplow/enrich/nsq/ConfigSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -158,7 +158,8 @@ class ConfigSpec extends Specification with CatsEffect { ) ), io.License(accept = true), - io.Validation(AtomicFields.from(atomicFieldLimitsDefaults ++ Map("app_id" -> 5, "mkt_clickid" -> 100000))) + io.Validation(AtomicFields.from(atomicFieldLimitsDefaults ++ Map("app_id" -> 5, "mkt_clickid" -> 100000))), + maxJsonDepth = 50 ) ConfigFile.parse[IO](configPath.asRight).value.map(result => result must beRight(expected)) } @@ -240,7 +241,8 @@ class ConfigSpec extends Specification with CatsEffect { adaptersSchemas, io.BlobStorageClients(gcs = true, s3 = true, azureStorage = None), io.License(accept = true), - io.Validation(AtomicFields.from(atomicFieldLimitsDefaults)) + io.Validation(AtomicFields.from(atomicFieldLimitsDefaults)), + maxJsonDepth = 40 ) ConfigFile.parse[IO](configPath.asRight).value.map(result => result must beRight(expected)) } diff --git a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Main.scala b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Main.scala index cb0bf67fb..65f5e73b0 100644 --- a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Main.scala +++ b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Main.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Sink.scala b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Sink.scala index 67d6372c7..c6b0665a0 100644 --- a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Sink.scala +++ b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Sink.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Source.scala b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Source.scala index 5bdb04614..3985d8c6f 100644 --- a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Source.scala +++ b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Source.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Utils.scala b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Utils.scala index a1d878185..ed24c75fd 100644 --- a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Utils.scala +++ b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/Utils.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/package.scala b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/package.scala index ba5c1947e..c2e019759 100644 --- a/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/package.scala +++ b/modules/pubsub/src/main/scala/com/snowplowanalytics/snowplow/enrich/pubsub/package.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/modules/pubsub/src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/ConfigSpec.scala b/modules/pubsub/src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/ConfigSpec.scala index 8765d91bc..06dafeda2 100644 --- a/modules/pubsub/src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/ConfigSpec.scala +++ b/modules/pubsub/src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/ConfigSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -132,7 +132,8 @@ class ConfigSpec extends Specification with CatsEffect { adaptersSchemas, io.BlobStorageClients(gcs = true, s3 = false, azureStorage = None), io.License(accept = true), - io.Validation(AtomicFields.from(atomicFieldLimitsDefaults ++ Map("app_id" -> 5, "mkt_clickid" -> 100000))) + io.Validation(AtomicFields.from(atomicFieldLimitsDefaults ++ Map("app_id" -> 5, "mkt_clickid" -> 100000))), + maxJsonDepth = 50 ) ConfigFile.parse[IO](configPath.asRight).value.map(result => result must beRight(expected)) } @@ -206,7 +207,8 @@ class ConfigSpec extends Specification with CatsEffect { adaptersSchemas, io.BlobStorageClients(gcs = true, s3 = false, azureStorage = None), io.License(accept = true), - io.Validation(AtomicFields.from(atomicFieldLimitsDefaults)) + io.Validation(AtomicFields.from(atomicFieldLimitsDefaults)), + maxJsonDepth = 40 ) ConfigFile.parse[IO](configPath.asRight).value.map(result => result must beRight(expected)) } diff --git a/modules/pubsub/src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/GcpUserAgentSpec.scala b/modules/pubsub/src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/GcpUserAgentSpec.scala index 37fce4981..d2b9b4066 100644 --- a/modules/pubsub/src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/GcpUserAgentSpec.scala +++ b/modules/pubsub/src/test/scala/com/snowplowanalytics/snowplow/enrich/pubsub/GcpUserAgentSpec.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index e6ca52360..9e4679f7b 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -29,7 +29,7 @@ object BuildSettings { lazy val projectSettings = Seq( organization := "com.snowplowanalytics", scalaVersion := "2.12.15", - licenses += ("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0.html")) + licenses += ("SLULA-1.1", url("https://docs.snowplow.io/limited-use-license-1.1")) ) lazy val commonProjectSettings = projectSettings ++ Seq( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 3cc643e41..880170e87 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -3,8 +3,8 @@ * All rights reserved. * * This software is made available by Snowplow Analytics, Ltd., - * under the terms of the Snowplow Limited Use License Agreement, Version 1.0 - * located at https://docs.snowplow.io/limited-use-license-1.0 + * under the terms of the Snowplow Limited Use License Agreement, Version 1.1 + * located at https://docs.snowplow.io/limited-use-license-1.1 * BY INSTALLING, DOWNLOADING, ACCESSING, USING OR DISTRIBUTING ANY PORTION * OF THE SOFTWARE, YOU AGREE TO THE TERMS OF SUCH LICENSE AGREEMENT. */ @@ -30,10 +30,11 @@ object Dependencies { // Java val commonsCodec = "1.16.0" val commonsText = "1.10.0" + val commonsIO = "2.14.0" val jodaTime = "2.10.1" val useragent = "1.21" val uaParser = "1.5.4" - val snakeYaml = "2.2" + val snakeYaml = "2.3" val postgresDriver = "42.7.2" val mysqlClient = "3.4.0" val hikariCP = "5.0.1" @@ -47,8 +48,8 @@ object Dependencies { val log4j = "2.20.0" val thrift = "0.15.0" val sprayJson = "1.3.6" - val netty = "4.1.108.Final" - val protobuf = "3.21.9" + val netty = "4.1.115.Final" + val protobuf = "4.28.3" val refererParser = "2.0.0" val maxmindIplookups = "0.8.1" @@ -61,30 +62,31 @@ object Dependencies { val gatlingJsonpath = "0.6.14" val scalaUri = "1.5.1" val badRows = "2.3.0" - val igluClient = "3.1.0" + val igluClient = "3.2.0" val snowplowRawEvent = "0.1.0" val collectorPayload = "0.0.0" val schemaSniffer = "0.0.0" - val awsSdk = "1.12.694" - val gcpSdk = "2.36.1" - val awsSdk2 = "2.25.24" + val awsSdk = "1.12.778" + val gcpSdk = "2.45.0" + val awsSdk2 = "2.29.20" val kinesisClient2 = "2.4.3" - val kafka = "3.7.0" + val kafka = "3.9.0" val mskAuth = "2.0.3" val nsqClient = "1.3.0" - val jackson = "2.16.1" + val jackson = "2.18.1" val config = "1.3.4" val decline = "1.0.0" val fs2 = "3.10.2" val catsEffect = "3.5.4" - val fs2PubSub = "0.22.0" + val fs2PubSub = "0.22.1" val fs2Aws = "4.1.0" - val fs2Kafka = "3.5.1" - val fs2BlobStorage = "0.9.12" - val azureIdentity = "1.11.1" + val fs2Kafka = "3.6.0" + val fs2BlobStorage = "0.9.15" + val azureIdentity = "1.12.2" + val msal4j = "1.17.3" val nimbusJoseJwt = "9.37.2" val http4s = "0.23.25" val log4cats = "2.6.0" @@ -109,6 +111,7 @@ object Dependencies { object Libraries { val commonsCodec = "commons-codec" % "commons-codec" % V.commonsCodec val commonsText = "org.apache.commons" % "commons-text" % V.commonsText + val commonsIO = "commons-io" % "commons-io" % V.commonsIO val jodaTime = "joda-time" % "joda-time" % V.jodaTime val useragent = "eu.bitwalker" % "UserAgentUtils" % V.useragent val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % V.jackson @@ -195,6 +198,7 @@ object Dependencies { .exclude("software.amazon.glue", "schema-registry-serde") val stsSdk2 = "software.amazon.awssdk" % "sts" % V.awsSdk2 % Runtime val azureIdentity = "com.azure" % "azure-identity" % V.azureIdentity + val msal4j = "com.microsoft.azure" % "msal4j" % V.msal4j val nimbusJoseJwt = "com.nimbusds" % "nimbus-jose-jwt" % V.nimbusJoseJwt val jacksonDfXml = "com.fasterxml.jackson.dataformat" % "jackson-dataformat-xml" % V.jackson val http4sClient = "org.http4s" %% "http4s-ember-client" % V.http4s @@ -216,6 +220,7 @@ object Dependencies { jodaTime, commonsCodec, commonsText, + commonsIO, // for security vulnerabilities useragent, jacksonDatabind, uaParser, @@ -309,6 +314,7 @@ object Dependencies { fs2BlobGcs, gcs, fs2PubSub, + protobuf, // for security vulnerabilities specs2, specs2CE ) @@ -331,7 +337,8 @@ object Dependencies { val kafkaDependencies = Seq( fs2Kafka, - kafkaClients, // override kafka-clients 2.8.1 from fs2Kafka to address https://security.snyk.io/vuln/SNYK-JAVA-ORGAPACHEKAFKA-3027430 + kafkaClients, // for security vulnerabilities + msal4j, // for security vulnerabilities mskAuth, specs2, specs2CE