Skip to content

Commit

Permalink
JPERF-1106: Gather Apache2 logs
Browse files Browse the repository at this point in the history
  • Loading branch information
pczuj committed May 18, 2023
1 parent c10ced8 commit df3bd05
Show file tree
Hide file tree
Showing 12 changed files with 275 additions and 50 deletions.
15 changes: 15 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,23 @@ Dropping a requirement of a major version of a dependency is a new contract.
## [Unreleased]
[Unreleased]: https://github.com/atlassian/aws-infrastructure/compare/release-2.29.0...master

## Added
- Add `DiagnosableLoadBalancer`.
- Add `Jira.Builder`.
- Add ability to `gatherDiagnostics` from `ApacheProxyLoadBalancer` by making it `DiagnosableLoadBalancer`.
- Make it possible to specify `extraResults` inside `Jira`, so that it's possible to `gatherResults` from any of Jira hosted integrations, e.g. load balancer or plugins.
- Let `Jira` produced by `DataCenterFormula` gather logs of `ApacheProxyLoadBalancer`. Resolve [JPERF-1106].
- Make `StartedNode` a `MeasurementSource`.

## Deprecated
- Deprecate `Jira` constructors in favor of `Jira.Builder`.

### Fixed
- Change default virtual user instance type to `c5.9xlarge`. It's better, cheaper and there seem to be availability issues with previous default (`c4.8xlarge`).
- Let every `MeasurementSource` inside `Jira` finish its gathering even if one of them fails. Fix [JPERF-1114].

[JPERF-1106]: https://ecosystem.atlassian.net/browse/JPERF-1106
[JPERF-1114]: https://ecosystem.atlassian.net/browse/JPERF-1114

## [2.29.0] - 2023-03-24
[2.29.0]: https://github.com/atlassian/aws-infrastructure/compare/release-2.28.0...release-2.29.0
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package com.atlassian.performance.tools.awsinfrastructure

internal class SuppressingRunnable(
private val delegates: Iterable<Runnable>
) : Runnable {
override fun run() {
val exceptions = delegates.fold(emptyList<Exception>()) { exceptions, runnable ->
val exception = try {
runnable.run()
null
} catch (e: Exception) {
e
}
exceptions + listOfNotNull(exception)
}

when {
exceptions.isEmpty() -> return
exceptions.size == 1 -> throw exceptions[0]
else -> {
val root = Exception("Multiple exceptions were thrown and are added suppressed into this one")
throw exceptions.fold(root) { root, it -> root.addSuppressed(it); root }
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import com.atlassian.performance.tools.awsinfrastructure.api.hardware.M4ExtraLar
import com.atlassian.performance.tools.awsinfrastructure.api.hardware.Volume
import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.ApacheEc2LoadBalancerFormula
import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.ApacheProxyLoadBalancer
import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.DiagnosableLoadBalancer
import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.LoadBalancerFormula
import com.atlassian.performance.tools.awsinfrastructure.api.network.Network
import com.atlassian.performance.tools.awsinfrastructure.api.network.NetworkFormula
Expand All @@ -20,6 +21,7 @@ import com.atlassian.performance.tools.awsinfrastructure.jira.DataCenterNodeForm
import com.atlassian.performance.tools.awsinfrastructure.jira.DiagnosableNodeFormula
import com.atlassian.performance.tools.awsinfrastructure.jira.StandaloneNodeFormula
import com.atlassian.performance.tools.awsinfrastructure.jira.home.SharedHomeFormula
import com.atlassian.performance.tools.awsinfrastructure.loadbalancer.LoadBalancerDiagnosticsSource.Extension.asMeasurementSource
import com.atlassian.performance.tools.concurrency.api.AbruptExecutorService
import com.atlassian.performance.tools.concurrency.api.submitWithLogContext
import com.atlassian.performance.tools.infrastructure.api.app.Apps
Expand Down Expand Up @@ -382,16 +384,18 @@ class DataCenterFormula private constructor(
loadBalancer.waitUntilHealthy(Duration.ofMinutes(5))
}

val jira = Jira(
nodes = nodes,
jiraHome = RemoteLocation(
sharedHomeSsh.host,
sharedHome.get().remoteSharedHome
),
database = databaseDataLocation,
address = loadBalancer.uri,
jmxClients = jiraNodes.mapIndexed { i, node -> configs[i].remoteJmx.getClient(node.publicIpAddress) }
val jiraHomeLocation = RemoteLocation(
sharedHomeSsh.host,
sharedHome.get().remoteSharedHome
)
val loadBalancerResultsSource = (provisionedLoadBalancer.loadBalancer as? DiagnosableLoadBalancer)
?.asMeasurementSource(resultsTransport.location)
val jira = Jira.Builder(address = loadBalancer.uri, jiraHome = jiraHomeLocation)
.database(databaseDataLocation)
.nodes(nodes)
.jmxClients(jiraNodes.mapIndexed { i, node -> configs[i].remoteJmx.getClient(node.publicIpAddress) })
.extraResults(listOfNotNull(loadBalancerResultsSource))
.build()
logger.info("$jira is set up, will expire ${jiraStack.expiry}")
return ProvisionedJira.Builder(jira)
.resource(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,41 +1,113 @@
package com.atlassian.performance.tools.awsinfrastructure.api.jira

import com.atlassian.performance.tools.awsinfrastructure.SuppressingRunnable
import com.atlassian.performance.tools.awsinfrastructure.api.RemoteLocation
import com.atlassian.performance.tools.concurrency.api.submitWithLogContext
import com.atlassian.performance.tools.infrastructure.api.MeasurementSource
import com.atlassian.performance.tools.infrastructure.api.jvm.jmx.JmxClient
import com.atlassian.performance.tools.jvmtasks.api.TaskTimer.time
import com.google.common.util.concurrent.ThreadFactoryBuilder
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import java.net.URI
import java.util.concurrent.Executors

class Jira(
/**
* @param extraResultsSources source of results/diagnostics of: reverse proxy, Crowd, LDAP, DVCS, DB, Jira plugins
* or anything that can be integrated with Jira as part of web application provisioning
*/
class Jira private constructor(
private val nodes: List<StartedNode>,
val jiraHome: RemoteLocation,
val database: RemoteLocation?,
val address: URI,
val jmxClients: List<JmxClient> = emptyList()
val jmxClients: List<JmxClient>,
private val extraResultsSources: List<MeasurementSource>
) : MeasurementSource {
private val logger: Logger = LogManager.getLogger(this::class.java)

@Deprecated("Use Jira.Builder instead.")
constructor(
nodes: List<StartedNode>,
jiraHome: RemoteLocation,
database: RemoteLocation?,
address: URI,
jmxClients: List<JmxClient>
) : this(
nodes = nodes,
jiraHome = jiraHome,
database = database,
address = address,
jmxClients = jmxClients,
extraResultsSources = emptyList()
)

@Deprecated("Use Jira.Builder instead.")
constructor(
nodes: List<StartedNode>,
jiraHome: RemoteLocation,
database: RemoteLocation?,
address: URI
) : this(
nodes = nodes,
jiraHome = jiraHome,
database = database,
address = address,
jmxClients = emptyList(),
extraResultsSources = emptyList()
)

override fun gatherResults() {
if (nodes.isEmpty()) {
logger.warn("No Jira nodes known to JPT, not downloading node results")
val firstNode = nodes.firstOrNull()
val measurementSources = extraResultsSources + nodes + listOfNotNull(firstNode?.let { AnalyticsLogsSource(it) })
if (measurementSources.isEmpty()) {
logger.warn("No result sources known to Jira, can't download anything")
return
}
val executor = Executors.newFixedThreadPool(
nodes.size.coerceAtMost(4),
measurementSources.size.coerceAtMost(4),
ThreadFactoryBuilder()
.setNameFormat("results-gathering-thread-%d")
.build()
)
nodes.map { executor.submitWithLogContext("gather $it") { it.gatherResults() } }
.forEach { it.get() }
time("gather analytics") { nodes.firstOrNull()?.gatherAnalyticLogs() }
SuppressingRunnable(
measurementSources.map { executor.submitWithLogContext("gather $it") { it.gatherResults() } }
.map { Runnable { it.get() } }
).run()

executor.shutdownNow()
}

override fun toString() = "Jira(address=$address)"

class Builder(
private val address: URI,
private val jiraHome: RemoteLocation
) {
private var database: RemoteLocation? = null
private var nodes: List<StartedNode> = emptyList()
private var jmxClients: List<JmxClient> = emptyList()
private var extraResults: List<MeasurementSource> = emptyList()

fun database(database: RemoteLocation?) = apply { this.database = database }
fun nodes(nodes: List<StartedNode>) = apply { this.nodes = nodes }
fun jmxClients(jmxClients: List<JmxClient>) = apply { this.jmxClients = jmxClients }
fun extraResults(extraResults: List<MeasurementSource>) = apply { this.extraResults = extraResults }

fun build() = Jira(
nodes = nodes,
jiraHome = jiraHome,
database = database,
address = address,
jmxClients = jmxClients,
extraResultsSources = extraResults
)
}

private class AnalyticsLogsSource(
private val node: StartedNode
) : MeasurementSource {
override fun gatherResults() {
node.gatherAnalyticLogs()
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -291,16 +291,15 @@ class StandaloneFormula private constructor(
logger.warn("It's possible that defined external access to Jira resources (e.g. http, debug, splunk) wasn't granted.")
}

val jira = Jira(
nodes = listOf(node),
jiraHome = RemoteLocation(
jiraSsh.host,
provisionedNode.jiraHome
),
database = databaseDataLocation,
address = jiraPublicHttpAddress,
jmxClients = listOf(config.remoteJmx.getClient(jiraPublicIp))
val jiraHomeLocation = RemoteLocation(
host = jiraSsh.host,
location = provisionedNode.jiraHome
)
val jira = Jira.Builder(address = jiraPublicHttpAddress, jiraHome = jiraHomeLocation)
.nodes(listOf(node))
.database(databaseDataLocation)
.jmxClients(listOf(config.remoteJmx.getClient(jiraPublicIp)))
.build()
logger.info("$jira is set up, will expire ${jiraStack.expiry}")
return ProvisionedJira.Builder(jira)
.resource(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package com.atlassian.performance.tools.awsinfrastructure.api.jira

import com.atlassian.performance.tools.aws.api.Storage
import com.atlassian.performance.tools.awsinfrastructure.api.aws.AwsCli
import com.atlassian.performance.tools.infrastructure.api.MeasurementSource
import com.atlassian.performance.tools.infrastructure.api.jira.JiraGcLog
import com.atlassian.performance.tools.infrastructure.api.process.RemoteMonitoringProcess
import com.atlassian.performance.tools.ssh.api.Ssh
Expand All @@ -15,10 +16,10 @@ class StartedNode(
private val unpackedProduct: String,
private val monitoringProcesses: List<RemoteMonitoringProcess>,
private val ssh: Ssh
) {
) : MeasurementSource {
private val resultsDirectory = "results"

fun gatherResults() {
override fun gatherResults() {
ssh.newConnection().use { shell ->
monitoringProcesses.forEach { it.stop(shell) }
val nodeResultsDirectory = "$resultsDirectory/'$name'"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package com.atlassian.performance.tools.awsinfrastructure.api.jira

import com.atlassian.performance.tools.aws.api.*
import com.atlassian.performance.tools.aws.api.Aws
import com.atlassian.performance.tools.aws.api.Investment
import com.atlassian.performance.tools.aws.api.SshKey
import com.atlassian.performance.tools.aws.api.Storage
import com.atlassian.performance.tools.awsinfrastructure.api.RemoteLocation
import com.atlassian.performance.tools.ssh.api.SshHost
import com.atlassian.performance.tools.ssh.api.auth.PasswordAuthentication
Expand All @@ -18,22 +21,17 @@ class UriJiraFormula(
key: Future<SshKey>,
roleProfile: String,
aws: Aws
): ProvisionedJira = ProvisionedJira
.Builder(
Jira(
nodes = emptyList(),
jiraHome = RemoteLocation(
host = SshHost(
ipAddress = "unknown",
userName = "unknown",
authentication = PasswordAuthentication("unknown"),
port = -1
),
location = "unknown"
),
database = null,
address = jiraAddress
)
): ProvisionedJira {
val jiraHomeLocation = RemoteLocation(
host = SshHost(
ipAddress = "unknown",
userName = "unknown",
authentication = PasswordAuthentication("unknown"),
port = -1
),
location = "unknown"
)
.build()
val jira = Jira.Builder(jiraAddress, jiraHomeLocation).build()
return ProvisionedJira.Builder(jira).build()
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
package com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer

import com.atlassian.performance.tools.aws.api.StorageLocation
import com.atlassian.performance.tools.awsinfrastructure.api.aws.AwsCli
import com.atlassian.performance.tools.infrastructure.api.Sed
import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer
import com.atlassian.performance.tools.jvmtasks.api.ExponentialBackoff
import com.atlassian.performance.tools.jvmtasks.api.IdempotentAction
import com.atlassian.performance.tools.ssh.api.Ssh
Expand All @@ -14,7 +15,7 @@ class ApacheProxyLoadBalancer private constructor(
private val ssh: Ssh,
ipAddress: String,
httpPort: Int
) : LoadBalancer {
) : DiagnosableLoadBalancer {

@Deprecated(message = "Use ApacheProxyLoadBalancer.Builder instead.")
constructor(
Expand Down Expand Up @@ -93,6 +94,15 @@ class ApacheProxyLoadBalancer private constructor(
connection.execute("echo \"$line\" | sudo tee -a $APACHE_CONFIG_PATH")
}

override fun gatherDiagnostics(location: StorageLocation) {
ssh.newConnection().use { connection ->
val resultsDir = "/tmp/s3-results"
connection.execute("mkdir -p $resultsDir")
connection.execute("cp -R /var/log/apache2 $resultsDir")
AwsCli().upload(location, connection, resultsDir, Duration.ofMinutes(1))
}
}

class Builder(
private val ssh: Ssh
) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer

import com.atlassian.performance.tools.aws.api.StorageLocation
import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer

interface DiagnosableLoadBalancer : LoadBalancer {
fun gatherDiagnostics(location: StorageLocation)
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import com.atlassian.performance.tools.aws.api.Resource
import com.atlassian.performance.tools.aws.api.UnallocatedResource
import com.atlassian.performance.tools.awsinfrastructure.api.network.access.AccessProvider
import com.atlassian.performance.tools.awsinfrastructure.api.network.access.AccessRequester
import com.atlassian.performance.tools.awsinfrastructure.api.network.access.NoAccessRequester
import com.atlassian.performance.tools.awsinfrastructure.api.network.access.NoAccessProvider
import com.atlassian.performance.tools.awsinfrastructure.api.network.access.NoAccessRequester
import com.atlassian.performance.tools.infrastructure.api.loadbalancer.LoadBalancer

class ProvisionedLoadBalancer private constructor(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package com.atlassian.performance.tools.awsinfrastructure.loadbalancer

import com.atlassian.performance.tools.aws.api.StorageLocation
import com.atlassian.performance.tools.awsinfrastructure.api.loadbalancer.DiagnosableLoadBalancer
import com.atlassian.performance.tools.infrastructure.api.MeasurementSource

internal class LoadBalancerDiagnosticsSource(
private val loadBalancer: DiagnosableLoadBalancer,
private val location: StorageLocation
) : MeasurementSource {
internal object Extension {
fun DiagnosableLoadBalancer.asMeasurementSource(
location: StorageLocation
) = LoadBalancerDiagnosticsSource(
loadBalancer = this,
location = location
)
}

override fun gatherResults() {
loadBalancer.gatherDiagnostics(location)
}
}
Loading

0 comments on commit df3bd05

Please sign in to comment.