Add variables to configure s3-sqs defaults for spark streaming. (#188)

* Add variables to configure s3-sqs defaults for spark streaming. * Update variables.tf Co-authored-by: Scott Barnhart <[email protected]> * Update VARIABLES.md Co-authored-by: Scott Barnhart <[email protected]> Co-authored-by: Raj Poluri <[email protected]> Co-authored-by: Scott Barnhart <[email protected]>
ExpediaGroup · Mar 4, 2021 · d6ea25f · d6ea25f
1 parent 25b27b4
commit d6ea25f
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.7.7] - 2021-03-03
+### Changed
+- Add variables to configure s3-sqs defaults for spark streaming.
+
 ## [6.7.6] - 2021-03-02
 ### Fixed
 - Disable k8s loadbalancer and route53 entries along with vpc endpoint services.

diff --git a/VARIABLES.md b/VARIABLES.md
@@ -69,6 +69,10 @@
 | s3_inventory_update_schedule | Cron schedule to update S3 inventory tables (if enabled). Defaults to every 12 hours. | string | `0 */12 * * *` | no |
 | s3_lifecycle_policy_transition_period | Number of days for transition to a different storage class using lifecycle policy. | string  | `30` | no |
 | s3_lifecycle_abort_incomplete_multipart_upload_days | Number of days after which incomplete multipart uploads will be deleted. | string  | `7` | no |
+| s3\_logs\_sqs\_delay\_seconds | The time in seconds that the delivery of all messages in the queue will be delayed. | `number` | `300` | no |
+| s3\_logs\_sqs\_message\_retention\_seconds | Time in seconds after which message will be deleted from the queue. | `number` | `345600` | no |
+| s3\_logs\_sqs\_receive\_wait\_time\_seconds | The time for which a ReceiveMessage call will wait for a message to arrive (long polling) before returning. | `number` | `10` | no |
+| s3\_logs\_sqs\_visibility\_timeout\_seconds | Time in seconds after which message will be returned to the queue if it is not deleted. | `number` | `3600` | no |
 | s3_storage_class | Destination S3 storage class for transition in the lifecycle policy. | string  | `INTELLIGENT_TIERING` | no |
 | secondary_vpcs | List of VPCs to associate with Service Discovery namespace. | list | `<list>` | no |
 | table_param_filter | A regular expression for selecting necessary table parameters for the SNS listener. If the value isn't set, then no table parameters are selected. | string | `` | no |

diff --git a/sns.tf b/sns.tf
@@ -81,6 +81,11 @@ resource "aws_sqs_queue" "apiary_managed_logs_queue" {
   name  = "${local.instance_alias}-s3-logs-queue"
   tags  = var.apiary_tags
 
+  visibility_timeout_seconds = var.s3_logs_sqs_visibility_timeout_seconds
+  message_retention_seconds  = var.s3_logs_sqs_message_retention_seconds
+  delay_seconds              = var.s3_logs_sqs_delay_seconds
+  receive_wait_time_seconds  = var.s3_logs_sqs_receive_wait_time_seconds
+
   policy = <<POLICY
 {
   "Version": "2012-10-17",

diff --git a/variables.tf b/variables.tf
@@ -66,6 +66,30 @@ variable "enable_apiary_s3_log_hive" {
   default     = true
 }
 
+variable "s3_logs_sqs_visibility_timeout_seconds" {
+  description = "Time in seconds after which message will be returned to the queue if it is not deleted."
+  type        = number
+  default     = 3600
+}
+
+variable "s3_logs_sqs_message_retention_seconds" {
+  description = "Time in seconds after which message will be deleted from the queue."
+  type        = number
+  default     = 345600
+}
+
+variable "s3_logs_sqs_delay_seconds" {
+  description = "The time in seconds that the delivery of all messages in the queue will be delayed."
+  type        = number
+  default     = 300
+}
+
+variable "s3_logs_sqs_receive_wait_time_seconds" {
+  description = "The time for which a ReceiveMessage call will wait for a message to arrive (long polling) before returning."
+  type        = number
+  default     = 10
+}
+
 variable "enable_hive_metastore_metrics" {
   description = "Enable sending Hive Metastore metrics to CloudWatch."
   type        = bool