From da690c67824081f72dcbfb91ceca63da6f39fd51 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Mon, 24 Jun 2024 16:25:42 -0600 Subject: [PATCH 01/43] modify which evem endpoint is being tested for performance testing --- .github/actions/performance-tests/art.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 71c1b3a15e..63b1046049 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -1,7 +1,7 @@ # artillery run art.yaml --output results.json # artillery report results.json --output artillery_report.html config: - target: "http://127.0.0.1:8545" + target: "http://localhost:9545" phases: - duration: 60 arrivalRate: 10 From c630c670dcae103f02aaca1faedac0f8da87810c Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Mon, 24 Jun 2024 17:03:49 -0600 Subject: [PATCH 02/43] modify which evem endpoint is being tested for performance testing --- .github/workflows/ci-nightly-performance-testing.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 30116f743f..334f8d722f 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -16,6 +16,17 @@ jobs: run: | make start-e2e-test + - name: "TEST:LOCAL:NET:ALIVE" + run: | + curl -X POST http://localhost:9545/ \ + -H "Content-Type: application/json" \ + -d '{ + "id": 1, + "jsonrpc": "2.0", + "method": "web3_clientVersion", + "params": [] + }' + - name: "EXECUTE:LOADTESTS" uses: artilleryio/action-cli@v1 with: From a7caddc3ab25e2c3d559d0e33195c7ad765d45b7 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 13:41:41 -0600 Subject: [PATCH 03/43] updated --- .github/actions/performance-tests/art.yaml | 546 ++++++++++----------- 1 file changed, 273 insertions(+), 273 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 63b1046049..69c0c75a25 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -28,276 +28,276 @@ scenarios: jsonrpc: "2.0" method: "web3_clientVersion" params: [] - - - name: web3_sha3 - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "web3_sha3" - params: - - "0x68656c6c6f20776f726c64" - - - name: net_version - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "net_version" - params: [] - - - name: net_listening - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "net_listening" - params: [] - - - name: net_peerCount - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "net_peerCount" - params: [] - - - name: eth_chainId - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_chainId" - params: [] - - - name: eth_getStorageAt - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getStorageAt" - params: - - "latest" - - - name: eth_getTransactionByBlockHashAndIndex - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getTransactionByBlockHashAndIndex" - params: - - "0x0" - - - name: eth_getTransactionByBlockNumberAndIndex - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getTransactionByBlockNumberAndIndex" - params: - - "latest" - - "0x0" - - - name: eth_getUncleByBlockHashAndIndex - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getUncleByBlockHashAndIndex" - params: - - "0x0" - - - name: eth_getUncleByBlockNumberAndIndex - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getUncleByBlockNumberAndIndex" - params: - - "latest" - - "0x0" - - - name: eth_newFilter - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_newFilter" - params: - - { - fromBlock: "latest", - toBlock: "latest", - address: "0x0", - topics: [] - } - - - name: eth_getFilterChanges - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getFilterChanges" - params: - - "0x0" - - - name: eth_getFilterLogs - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getFilterLogs" - params: - - "0x0" - - - name: eth_blockNumber - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_blockNumber" - params: [] - - - name: eth_getBlockByHash - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getBlockByHash" - params: - - "0x0" - - true - - - name: eth_getBlockByNumber - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getBlockByNumber" - params: - - "latest" - - true - - - name: eth_getBlockTransactionCountByHash - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getBlockTransactionCountByHash" - params: - - "0x0" - - - name: eth_getBlockTransactionCountByNumber - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getBlockTransactionCountByNumber" - params: - - "latest" - - - name: eth_getCode - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getCode" - params: - - "0x0" - - "latest" - - - name: eth_getTransactionByHash - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getTransactionByHash" - params: - - "0x0" - - - name: eth_getTransactionReceipt - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getTransactionReceipt" - params: - - "0x0" - - - name: eth_getLogs - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getLogs" - params: - - fromBlock: "0x1" - toBlock: "latest" - address: "0x0" - topics: ["0x0"] - - - name: eth_getBalance - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_getBalance" - params: - - "0x0" - - "latest" - - - name: eth_estimateGas - flow: - - post: - url: "/" - json: - id: 1 - jsonrpc: "2.0" - method: "eth_estimateGas" - params: - - from: "0x0" +# +# - name: web3_sha3 +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "web3_sha3" +# params: +# - "0x68656c6c6f20776f726c64" +# +# - name: net_version +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "net_version" +# params: [] +# +# - name: net_listening +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "net_listening" +# params: [] +# +# - name: net_peerCount +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "net_peerCount" +# params: [] +# +# - name: eth_chainId +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_chainId" +# params: [] +# +# - name: eth_getStorageAt +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getStorageAt" +# params: +# - "latest" +# +# - name: eth_getTransactionByBlockHashAndIndex +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getTransactionByBlockHashAndIndex" +# params: +# - "0x0" +# +# - name: eth_getTransactionByBlockNumberAndIndex +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getTransactionByBlockNumberAndIndex" +# params: +# - "latest" +# - "0x0" +# +# - name: eth_getUncleByBlockHashAndIndex +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getUncleByBlockHashAndIndex" +# params: +# - "0x0" +# +# - name: eth_getUncleByBlockNumberAndIndex +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getUncleByBlockNumberAndIndex" +# params: +# - "latest" +# - "0x0" +# +# - name: eth_newFilter +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_newFilter" +# params: +# - { +# fromBlock: "latest", +# toBlock: "latest", +# address: "0x0", +# topics: [] +# } +# +# - name: eth_getFilterChanges +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getFilterChanges" +# params: +# - "0x0" +# +# - name: eth_getFilterLogs +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getFilterLogs" +# params: +# - "0x0" +# +# - name: eth_blockNumber +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_blockNumber" +# params: [] +# +# - name: eth_getBlockByHash +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getBlockByHash" +# params: +# - "0x0" +# - true +# +# - name: eth_getBlockByNumber +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getBlockByNumber" +# params: +# - "latest" +# - true +# +# - name: eth_getBlockTransactionCountByHash +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getBlockTransactionCountByHash" +# params: +# - "0x0" +# +# - name: eth_getBlockTransactionCountByNumber +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getBlockTransactionCountByNumber" +# params: +# - "latest" +# +# - name: eth_getCode +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getCode" +# params: +# - "0x0" +# - "latest" +# +# - name: eth_getTransactionByHash +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getTransactionByHash" +# params: +# - "0x0" +# +# - name: eth_getTransactionReceipt +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getTransactionReceipt" +# params: +# - "0x0" +# +# - name: eth_getLogs +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getLogs" +# params: +# - fromBlock: "0x1" +# toBlock: "latest" +# address: "0x0" +# topics: ["0x0"] +# +# - name: eth_getBalance +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_getBalance" +# params: +# - "0x0" +# - "latest" +# +# - name: eth_estimateGas +# flow: +# - post: +# url: "/" +# json: +# id: 1 +# jsonrpc: "2.0" +# method: "eth_estimateGas" +# params: +# - from: "0x0" From 6cc58e484238db954247bfaa2be5a321525ead5e Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 13:51:23 -0600 Subject: [PATCH 04/43] updated --- .github/actions/performance-tests/art.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 69c0c75a25..4bb0ed2766 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -17,6 +17,7 @@ config: reports: - type: "html" filename: "artillery_report.html" + logLevel: debug scenarios: - name: web3_clientVersion From 422ee2bbb867410afddfbc501f6e20655e1b0873 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 14:11:06 -0600 Subject: [PATCH 05/43] updated --- .github/actions/performance-tests/art.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 4bb0ed2766..5c21257bb7 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -1,7 +1,7 @@ # artillery run art.yaml --output results.json # artillery report results.json --output artillery_report.html config: - target: "http://localhost:9545" + target: "http://127.0.0.1:9545" phases: - duration: 60 arrivalRate: 10 @@ -17,7 +17,8 @@ config: reports: - type: "html" filename: "artillery_report.html" - logLevel: debug + logLevel: debug + scenarios: - name: web3_clientVersion @@ -28,7 +29,8 @@ scenarios: id: 1 jsonrpc: "2.0" method: "web3_clientVersion" - params: [] + params: [ ] + # # - name: web3_sha3 # flow: From 3a729885e40c0327f77259773c223fb6acdac46c Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 14:15:27 -0600 Subject: [PATCH 06/43] updated --- .github/actions/performance-tests/art.yaml | 1 - .../ci-nightly-performance-testing.yaml | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 5c21257bb7..2f32980596 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -19,7 +19,6 @@ config: filename: "artillery_report.html" logLevel: debug - scenarios: - name: web3_clientVersion flow: diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 334f8d722f..d3d8f73f78 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -12,6 +12,11 @@ jobs: steps: - uses: actions/checkout@v4 + - name: "INSTALL:NODEJS" + uses: actions/setup-node@v4 + with: + node-version: 16 + - name: "START:LOCAL:NET" run: | make start-e2e-test @@ -27,15 +32,17 @@ jobs: "params": [] }' - - name: "EXECUTE:LOADTESTS" - uses: artilleryio/action-cli@v1 - with: - command: run .github/actions/performance-tests/art.yaml --record --key ${{ secrets.ARTILLERY_KEY }} --output ./report.json + - name: "INSTALL:ARTILLERY" + run: | + npm install -g artillery@latest + + - name: "EXECUTE:PERFORMANCE:TESTS" + run: | + artillery run .github/actions/performance-tests/art.yaml --record --key ${{ secrets.ARTILLERY_KEY }} --output ./report.json - name: "GENERATE:REPORT" - uses: artilleryio/action-cli@v1 - with: - command: report report.json --output artillery_report.html + run: | + artillery report report.json --output artillery_report.html - name: "UPLOAD:REPORT" uses: actions/upload-artifact@v3 From 7d2b613162715bae9ee8826eb87fc5bc7e66306b Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 14:15:38 -0600 Subject: [PATCH 07/43] updated --- .github/actions/performance-tests/art.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 2f32980596..685c91f7f0 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -1,7 +1,7 @@ # artillery run art.yaml --output results.json # artillery report results.json --output artillery_report.html config: - target: "http://127.0.0.1:9545" + target: "http://localhost:9545" phases: - duration: 60 arrivalRate: 10 From 7339d69270dfff989ae95d5a734b599e8b7f7e79 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 14:19:02 -0600 Subject: [PATCH 08/43] updated --- .github/actions/performance-tests/art.yaml | 3 +++ .../ci-nightly-performance-testing.yaml | 20 +++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 685c91f7f0..1035bdd3df 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -13,6 +13,9 @@ config: plugins: metrics-by-endpoint: useOnlyRequestNames: true + ensure: + - type: "failure" + threshold: 1 summary: true reports: - type: "html" diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index d3d8f73f78..4655dd696e 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -39,14 +39,30 @@ jobs: - name: "EXECUTE:PERFORMANCE:TESTS" run: | artillery run .github/actions/performance-tests/art.yaml --record --key ${{ secrets.ARTILLERY_KEY }} --output ./report.json + if [ $? -ne 0 ]; then + echo "Performance Testing had Failed Tests." + exit 1 + else + echo "EVM Performance Testing Successful" + fi - name: "GENERATE:REPORT" + if: always() run: | artillery report report.json --output artillery_report.html - name: "UPLOAD:REPORT" uses: actions/upload-artifact@v3 - if: success() + if: always() with: name: artillery-report - path: ./artillery_report.html \ No newline at end of file + path: ./artillery_report.html + + - name: "NOTIFY:SLACK:FAILURE" + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }} \ No newline at end of file From f3751a1aced150e273043ad51f2b642a7480fe15 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 14:47:32 -0600 Subject: [PATCH 09/43] updated --- .github/actions/performance-tests/art.yaml | 545 ++++++++++----------- 1 file changed, 272 insertions(+), 273 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 1035bdd3df..a461b43a65 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -33,276 +33,275 @@ scenarios: method: "web3_clientVersion" params: [ ] -# -# - name: web3_sha3 -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "web3_sha3" -# params: -# - "0x68656c6c6f20776f726c64" -# -# - name: net_version -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "net_version" -# params: [] -# -# - name: net_listening -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "net_listening" -# params: [] -# -# - name: net_peerCount -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "net_peerCount" -# params: [] -# -# - name: eth_chainId -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_chainId" -# params: [] -# -# - name: eth_getStorageAt -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getStorageAt" -# params: -# - "latest" -# -# - name: eth_getTransactionByBlockHashAndIndex -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getTransactionByBlockHashAndIndex" -# params: -# - "0x0" -# -# - name: eth_getTransactionByBlockNumberAndIndex -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getTransactionByBlockNumberAndIndex" -# params: -# - "latest" -# - "0x0" -# -# - name: eth_getUncleByBlockHashAndIndex -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getUncleByBlockHashAndIndex" -# params: -# - "0x0" -# -# - name: eth_getUncleByBlockNumberAndIndex -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getUncleByBlockNumberAndIndex" -# params: -# - "latest" -# - "0x0" -# -# - name: eth_newFilter -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_newFilter" -# params: -# - { -# fromBlock: "latest", -# toBlock: "latest", -# address: "0x0", -# topics: [] -# } -# -# - name: eth_getFilterChanges -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getFilterChanges" -# params: -# - "0x0" -# -# - name: eth_getFilterLogs -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getFilterLogs" -# params: -# - "0x0" -# -# - name: eth_blockNumber -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_blockNumber" -# params: [] -# -# - name: eth_getBlockByHash -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getBlockByHash" -# params: -# - "0x0" -# - true -# -# - name: eth_getBlockByNumber -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getBlockByNumber" -# params: -# - "latest" -# - true -# -# - name: eth_getBlockTransactionCountByHash -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getBlockTransactionCountByHash" -# params: -# - "0x0" -# -# - name: eth_getBlockTransactionCountByNumber -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getBlockTransactionCountByNumber" -# params: -# - "latest" -# -# - name: eth_getCode -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getCode" -# params: -# - "0x0" -# - "latest" -# -# - name: eth_getTransactionByHash -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getTransactionByHash" -# params: -# - "0x0" -# -# - name: eth_getTransactionReceipt -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getTransactionReceipt" -# params: -# - "0x0" -# -# - name: eth_getLogs -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getLogs" -# params: -# - fromBlock: "0x1" -# toBlock: "latest" -# address: "0x0" -# topics: ["0x0"] -# -# - name: eth_getBalance -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_getBalance" -# params: -# - "0x0" -# - "latest" -# -# - name: eth_estimateGas -# flow: -# - post: -# url: "/" -# json: -# id: 1 -# jsonrpc: "2.0" -# method: "eth_estimateGas" -# params: -# - from: "0x0" + - name: web3_sha3 + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "web3_sha3" + params: + - "0x68656c6c6f20776f726c64" + + - name: net_version + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "net_version" + params: [] + + - name: net_listening + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "net_listening" + params: [] + + - name: net_peerCount + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "net_peerCount" + params: [] + + - name: eth_chainId + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_chainId" + params: [] + + - name: eth_getStorageAt + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getStorageAt" + params: + - "latest" + + - name: eth_getTransactionByBlockHashAndIndex + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getTransactionByBlockHashAndIndex" + params: + - "0x0" + + - name: eth_getTransactionByBlockNumberAndIndex + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getTransactionByBlockNumberAndIndex" + params: + - "latest" + - "0x0" + + - name: eth_getUncleByBlockHashAndIndex + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getUncleByBlockHashAndIndex" + params: + - "0x0" + + - name: eth_getUncleByBlockNumberAndIndex + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getUncleByBlockNumberAndIndex" + params: + - "latest" + - "0x0" + + - name: eth_newFilter + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_newFilter" + params: + - { + fromBlock: "latest", + toBlock: "latest", + address: "0x0", + topics: [] + } + + - name: eth_getFilterChanges + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getFilterChanges" + params: + - "0x0" + + - name: eth_getFilterLogs + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getFilterLogs" + params: + - "0x0" + + - name: eth_blockNumber + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_blockNumber" + params: [] + + - name: eth_getBlockByHash + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getBlockByHash" + params: + - "0x0" + - true + + - name: eth_getBlockByNumber + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getBlockByNumber" + params: + - "latest" + - true + + - name: eth_getBlockTransactionCountByHash + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getBlockTransactionCountByHash" + params: + - "0x0" + + - name: eth_getBlockTransactionCountByNumber + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getBlockTransactionCountByNumber" + params: + - "latest" + + - name: eth_getCode + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getCode" + params: + - "0x0" + - "latest" + + - name: eth_getTransactionByHash + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getTransactionByHash" + params: + - "0x0" + + - name: eth_getTransactionReceipt + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getTransactionReceipt" + params: + - "0x0" + + - name: eth_getLogs + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getLogs" + params: + - fromBlock: "0x1" + toBlock: "latest" + address: "0x0" + topics: ["0x0"] + + - name: eth_getBalance + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_getBalance" + params: + - "0x0" + - "latest" + + - name: eth_estimateGas + flow: + - post: + url: "/" + json: + id: 1 + jsonrpc: "2.0" + method: "eth_estimateGas" + params: + - from: "0x0" From d595b5f826ae1c1d63806dc2fdfe23c7c146a2a6 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 14:51:33 -0600 Subject: [PATCH 10/43] updated --- .github/actions/performance-tests/art.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index a461b43a65..ddf3e67d84 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -26,7 +26,7 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/" + url: "/getit" json: id: 1 jsonrpc: "2.0" From 8e43a247329039794a373794127552d75a115c9a Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 14:57:14 -0600 Subject: [PATCH 11/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index ddf3e67d84..d6ce7d1c82 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -26,11 +26,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/getit" + url: "/" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersion" + method: "web3_clientVersionxxxxx" params: [ ] - name: web3_sha3 From 7187c4b62a469d9cf89a8fad90ca0f774ba23d02 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 15:06:10 -0600 Subject: [PATCH 12/43] updated --- .github/actions/performance-tests/art.yaml | 83 ++++++++++++++++++++-- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index d6ce7d1c82..1108d69cc1 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -1,5 +1,3 @@ -# artillery run art.yaml --output results.json -# artillery report results.json --output artillery_report.html config: target: "http://localhost:9545" phases: @@ -26,12 +24,15 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/" + url: "/sdfas" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersionxxxxx" - params: [ ] + method: "web3_clientVersionxxxxx" # Deliberate typo to force failure + params: [] + - ensure: + check: "statusCode == 200" + message: "web3_clientVersion request failed" - name: web3_sha3 flow: @@ -43,6 +44,9 @@ scenarios: method: "web3_sha3" params: - "0x68656c6c6f20776f726c64" + - ensure: + check: "statusCode == 200" + message: "web3_sha3 request failed" - name: net_version flow: @@ -53,6 +57,9 @@ scenarios: jsonrpc: "2.0" method: "net_version" params: [] + - ensure: + check: "statusCode == 200" + message: "net_version request failed" - name: net_listening flow: @@ -63,6 +70,9 @@ scenarios: jsonrpc: "2.0" method: "net_listening" params: [] + - ensure: + check: "statusCode == 200" + message: "net_listening request failed" - name: net_peerCount flow: @@ -73,6 +83,9 @@ scenarios: jsonrpc: "2.0" method: "net_peerCount" params: [] + - ensure: + check: "statusCode == 200" + message: "net_peerCount request failed" - name: eth_chainId flow: @@ -83,6 +96,9 @@ scenarios: jsonrpc: "2.0" method: "eth_chainId" params: [] + - ensure: + check: "statusCode == 200" + message: "eth_chainId request failed" - name: eth_getStorageAt flow: @@ -94,6 +110,9 @@ scenarios: method: "eth_getStorageAt" params: - "latest" + - ensure: + check: "statusCode == 200" + message: "eth_getStorageAt request failed" - name: eth_getTransactionByBlockHashAndIndex flow: @@ -105,6 +124,9 @@ scenarios: method: "eth_getTransactionByBlockHashAndIndex" params: - "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_getTransactionByBlockHashAndIndex request failed" - name: eth_getTransactionByBlockNumberAndIndex flow: @@ -117,6 +139,9 @@ scenarios: params: - "latest" - "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_getTransactionByBlockNumberAndIndex request failed" - name: eth_getUncleByBlockHashAndIndex flow: @@ -128,6 +153,9 @@ scenarios: method: "eth_getUncleByBlockHashAndIndex" params: - "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_getUncleByBlockHashAndIndex request failed" - name: eth_getUncleByBlockNumberAndIndex flow: @@ -140,6 +168,9 @@ scenarios: params: - "latest" - "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_getUncleByBlockNumberAndIndex request failed" - name: eth_newFilter flow: @@ -156,6 +187,9 @@ scenarios: address: "0x0", topics: [] } + - ensure: + check: "statusCode == 200" + message: "eth_newFilter request failed" - name: eth_getFilterChanges flow: @@ -167,6 +201,9 @@ scenarios: method: "eth_getFilterChanges" params: - "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_getFilterChanges request failed" - name: eth_getFilterLogs flow: @@ -178,6 +215,9 @@ scenarios: method: "eth_getFilterLogs" params: - "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_getFilterLogs request failed" - name: eth_blockNumber flow: @@ -188,6 +228,9 @@ scenarios: jsonrpc: "2.0" method: "eth_blockNumber" params: [] + - ensure: + check: "statusCode == 200" + message: "eth_blockNumber request failed" - name: eth_getBlockByHash flow: @@ -200,6 +243,9 @@ scenarios: params: - "0x0" - true + - ensure: + check: "statusCode == 200" + message: "eth_getBlockByHash request failed" - name: eth_getBlockByNumber flow: @@ -212,6 +258,9 @@ scenarios: params: - "latest" - true + - ensure: + check: "statusCode == 200" + message: "eth_getBlockByNumber request failed" - name: eth_getBlockTransactionCountByHash flow: @@ -223,6 +272,9 @@ scenarios: method: "eth_getBlockTransactionCountByHash" params: - "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_getBlockTransactionCountByHash request failed" - name: eth_getBlockTransactionCountByNumber flow: @@ -234,6 +286,9 @@ scenarios: method: "eth_getBlockTransactionCountByNumber" params: - "latest" + - ensure: + check: "statusCode == 200" + message: "eth_getBlockTransactionCountByNumber request failed" - name: eth_getCode flow: @@ -246,6 +301,9 @@ scenarios: params: - "0x0" - "latest" + - ensure: + check: "statusCode == 200" + message: "eth_getCode request failed" - name: eth_getTransactionByHash flow: @@ -257,6 +315,9 @@ scenarios: method: "eth_getTransactionByHash" params: - "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_getTransactionByHash request failed" - name: eth_getTransactionReceipt flow: @@ -268,6 +329,9 @@ scenarios: method: "eth_getTransactionReceipt" params: - "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_getTransactionReceipt request failed" - name: eth_getLogs flow: @@ -282,6 +346,9 @@ scenarios: toBlock: "latest" address: "0x0" topics: ["0x0"] + - ensure: + check: "statusCode == 200" + message: "eth_getLogs request failed" - name: eth_getBalance flow: @@ -294,6 +361,9 @@ scenarios: params: - "0x0" - "latest" + - ensure: + check: "statusCode == 200" + message: "eth_getBalance request failed" - name: eth_estimateGas flow: @@ -305,3 +375,6 @@ scenarios: method: "eth_estimateGas" params: - from: "0x0" + - ensure: + check: "statusCode == 200" + message: "eth_estimateGas request failed" From 7c60959d13ab9ef5627cd282541b971134f83801 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 15:15:29 -0600 Subject: [PATCH 13/43] updated --- .github/actions/performance-tests/art.yaml | 2 +- .github/workflows/ci-nightly-performance-testing.yaml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 1108d69cc1..31f79a8723 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,7 +24,7 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/sdfas" + url: "/xxxxx" json: id: 1 jsonrpc: "2.0" diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 4655dd696e..4268bf04d5 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -39,7 +39,16 @@ jobs: - name: "EXECUTE:PERFORMANCE:TESTS" run: | artillery run .github/actions/performance-tests/art.yaml --record --key ${{ secrets.ARTILLERY_KEY }} --output ./report.json + # Check Artillery exit status if [ $? -ne 0 ]; then + echo "Artillery command failed to execute." + exit 1 + fi + + # Parse the results.json file to check for failures + failures=$(jq '.errors.ECONNREFUSED' ./report.json) + + if [ "$failures" -gt 0 ]; then echo "Performance Testing had Failed Tests." exit 1 else From 4e9c8e1bacd380fb62a23130a7d48726fe47e7bf Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 15:33:48 -0600 Subject: [PATCH 14/43] updated --- .github/workflows/ci-nightly-performance-testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 4268bf04d5..4451d3b815 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -46,7 +46,7 @@ jobs: fi # Parse the results.json file to check for failures - failures=$(jq '.errors.ECONNREFUSED' ./report.json) + failures=$(jq '.errors.ECONNREFUSED // 0' ./report.json) if [ "$failures" -gt 0 ]; then echo "Performance Testing had Failed Tests." From a836facecd26ec29aa027dba3481dc3351904cc0 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 15:39:14 -0600 Subject: [PATCH 15/43] updated --- .github/workflows/ci-nightly-performance-testing.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 4451d3b815..a2f10845df 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -55,6 +55,16 @@ jobs: echo "EVM Performance Testing Successful" fi + # Parse the results.json file to check for failed vusers + failed_vusers=$(jq '.vusers.failed // 0' ./report.json) + + if [ "$failed_vusers" -gt 0 ]; then + echo "Performance Testing had Failed Tests." + exit 1 + else + echo "EVM Performance Testing Successful" + fi + - name: "GENERATE:REPORT" if: always() run: | From c79aca19e6a66fdf9de236263125c7ab6dd42249 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 15:45:43 -0600 Subject: [PATCH 16/43] updated --- .github/workflows/ci-nightly-performance-testing.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index a2f10845df..8f4885bd8e 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -45,6 +45,8 @@ jobs: exit 1 fi + cat report.json + # Parse the results.json file to check for failures failures=$(jq '.errors.ECONNREFUSED // 0' ./report.json) From 7824308ac7128ea001a5de928931587d30a09043 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 15:53:54 -0600 Subject: [PATCH 17/43] updated --- .../ci-nightly-performance-testing.yaml | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 8f4885bd8e..4ead4af6b5 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -45,22 +45,12 @@ jobs: exit 1 fi - cat report.json + # Parse the results.json file to check for failed vusers and http response codes + failed_vusers=$(jq '.aggregate.counters["vusers.failed"] // 0' ./report.json) + http_codes_200=$(jq '.aggregate.counters["http.codes.200"] // 0' ./report.json) + http_responses=$(jq '.aggregate.counters["http.responses"] // 0' ./report.json) - # Parse the results.json file to check for failures - failures=$(jq '.errors.ECONNREFUSED // 0' ./report.json) - - if [ "$failures" -gt 0 ]; then - echo "Performance Testing had Failed Tests." - exit 1 - else - echo "EVM Performance Testing Successful" - fi - - # Parse the results.json file to check for failed vusers - failed_vusers=$(jq '.vusers.failed // 0' ./report.json) - - if [ "$failed_vusers" -gt 0 ]; then + if [ "$failed_vusers" -gt 1 ] || [ "$http_codes_200" -ne "$http_responses" ]; then echo "Performance Testing had Failed Tests." exit 1 else From fad1e24ad6a3d5b81ac6444af93fcfaf9a3b395b Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 15:59:26 -0600 Subject: [PATCH 18/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- .github/workflows/ci-nightly-performance-testing.yaml | 11 ----------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 31f79a8723..ba8e15d6d5 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/xxxxx" + url: "/" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersionxxxxx" # Deliberate typo to force failure + method: "web3_clientVersion" # Deliberate typo to force failure params: [] - ensure: check: "statusCode == 200" diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 4ead4af6b5..69ecde8b73 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -21,17 +21,6 @@ jobs: run: | make start-e2e-test - - name: "TEST:LOCAL:NET:ALIVE" - run: | - curl -X POST http://localhost:9545/ \ - -H "Content-Type: application/json" \ - -d '{ - "id": 1, - "jsonrpc": "2.0", - "method": "web3_clientVersion", - "params": [] - }' - - name: "INSTALL:ARTILLERY" run: | npm install -g artillery@latest From c72efbf12fe7abe0ccd090fd6fc57fab4f426cdd Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 16:02:09 -0600 Subject: [PATCH 19/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- .github/workflows/ci-nightly-performance-testing.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index ba8e15d6d5..bb7a087cd7 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/" + url: "/sss" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersion" # Deliberate typo to force failure + method: "web3_clientVersionsss" # Deliberate typo to force failure params: [] - ensure: check: "statusCode == 200" diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 69ecde8b73..46d5b6fa5c 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -65,4 +65,4 @@ jobs: status: ${{ job.status }} fields: repo,message,commit,author,action,eventName,ref,workflow,job,took env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }} \ No newline at end of file + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_INTEGRATION_TESTING_WEBHOOK }} \ No newline at end of file From 71653cd0bc909501ea95fd939f9e412f658f366e Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 16:02:32 -0600 Subject: [PATCH 20/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index bb7a087cd7..ba8e15d6d5 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/sss" + url: "/" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersionsss" # Deliberate typo to force failure + method: "web3_clientVersion" # Deliberate typo to force failure params: [] - ensure: check: "statusCode == 200" From f620beaa1a98c4f25267830b45b2a83801cd126b Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 16:09:34 -0600 Subject: [PATCH 21/43] updated --- .github/workflows/ci-nightly-performance-testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 46d5b6fa5c..69ecde8b73 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -65,4 +65,4 @@ jobs: status: ${{ job.status }} fields: repo,message,commit,author,action,eventName,ref,workflow,job,took env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_INTEGRATION_TESTING_WEBHOOK }} \ No newline at end of file + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }} \ No newline at end of file From 0933e4b781ab9b1321237f7a0ea8c62e86750fa7 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 16:16:48 -0600 Subject: [PATCH 22/43] updated --- .github/actions/performance-tests/art.yaml | 75 ---------------------- 1 file changed, 75 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index ba8e15d6d5..edc2e4e2a5 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -30,9 +30,6 @@ scenarios: jsonrpc: "2.0" method: "web3_clientVersion" # Deliberate typo to force failure params: [] - - ensure: - check: "statusCode == 200" - message: "web3_clientVersion request failed" - name: web3_sha3 flow: @@ -44,9 +41,6 @@ scenarios: method: "web3_sha3" params: - "0x68656c6c6f20776f726c64" - - ensure: - check: "statusCode == 200" - message: "web3_sha3 request failed" - name: net_version flow: @@ -57,9 +51,6 @@ scenarios: jsonrpc: "2.0" method: "net_version" params: [] - - ensure: - check: "statusCode == 200" - message: "net_version request failed" - name: net_listening flow: @@ -70,9 +61,6 @@ scenarios: jsonrpc: "2.0" method: "net_listening" params: [] - - ensure: - check: "statusCode == 200" - message: "net_listening request failed" - name: net_peerCount flow: @@ -83,9 +71,6 @@ scenarios: jsonrpc: "2.0" method: "net_peerCount" params: [] - - ensure: - check: "statusCode == 200" - message: "net_peerCount request failed" - name: eth_chainId flow: @@ -96,9 +81,6 @@ scenarios: jsonrpc: "2.0" method: "eth_chainId" params: [] - - ensure: - check: "statusCode == 200" - message: "eth_chainId request failed" - name: eth_getStorageAt flow: @@ -110,9 +92,6 @@ scenarios: method: "eth_getStorageAt" params: - "latest" - - ensure: - check: "statusCode == 200" - message: "eth_getStorageAt request failed" - name: eth_getTransactionByBlockHashAndIndex flow: @@ -124,9 +103,6 @@ scenarios: method: "eth_getTransactionByBlockHashAndIndex" params: - "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_getTransactionByBlockHashAndIndex request failed" - name: eth_getTransactionByBlockNumberAndIndex flow: @@ -139,9 +115,6 @@ scenarios: params: - "latest" - "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_getTransactionByBlockNumberAndIndex request failed" - name: eth_getUncleByBlockHashAndIndex flow: @@ -153,9 +126,6 @@ scenarios: method: "eth_getUncleByBlockHashAndIndex" params: - "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_getUncleByBlockHashAndIndex request failed" - name: eth_getUncleByBlockNumberAndIndex flow: @@ -168,9 +138,6 @@ scenarios: params: - "latest" - "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_getUncleByBlockNumberAndIndex request failed" - name: eth_newFilter flow: @@ -187,9 +154,6 @@ scenarios: address: "0x0", topics: [] } - - ensure: - check: "statusCode == 200" - message: "eth_newFilter request failed" - name: eth_getFilterChanges flow: @@ -201,9 +165,6 @@ scenarios: method: "eth_getFilterChanges" params: - "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_getFilterChanges request failed" - name: eth_getFilterLogs flow: @@ -215,9 +176,6 @@ scenarios: method: "eth_getFilterLogs" params: - "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_getFilterLogs request failed" - name: eth_blockNumber flow: @@ -228,9 +186,6 @@ scenarios: jsonrpc: "2.0" method: "eth_blockNumber" params: [] - - ensure: - check: "statusCode == 200" - message: "eth_blockNumber request failed" - name: eth_getBlockByHash flow: @@ -243,9 +198,6 @@ scenarios: params: - "0x0" - true - - ensure: - check: "statusCode == 200" - message: "eth_getBlockByHash request failed" - name: eth_getBlockByNumber flow: @@ -258,9 +210,6 @@ scenarios: params: - "latest" - true - - ensure: - check: "statusCode == 200" - message: "eth_getBlockByNumber request failed" - name: eth_getBlockTransactionCountByHash flow: @@ -272,9 +221,6 @@ scenarios: method: "eth_getBlockTransactionCountByHash" params: - "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_getBlockTransactionCountByHash request failed" - name: eth_getBlockTransactionCountByNumber flow: @@ -286,9 +232,6 @@ scenarios: method: "eth_getBlockTransactionCountByNumber" params: - "latest" - - ensure: - check: "statusCode == 200" - message: "eth_getBlockTransactionCountByNumber request failed" - name: eth_getCode flow: @@ -301,9 +244,6 @@ scenarios: params: - "0x0" - "latest" - - ensure: - check: "statusCode == 200" - message: "eth_getCode request failed" - name: eth_getTransactionByHash flow: @@ -315,9 +255,6 @@ scenarios: method: "eth_getTransactionByHash" params: - "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_getTransactionByHash request failed" - name: eth_getTransactionReceipt flow: @@ -329,9 +266,6 @@ scenarios: method: "eth_getTransactionReceipt" params: - "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_getTransactionReceipt request failed" - name: eth_getLogs flow: @@ -346,9 +280,6 @@ scenarios: toBlock: "latest" address: "0x0" topics: ["0x0"] - - ensure: - check: "statusCode == 200" - message: "eth_getLogs request failed" - name: eth_getBalance flow: @@ -361,9 +292,6 @@ scenarios: params: - "0x0" - "latest" - - ensure: - check: "statusCode == 200" - message: "eth_getBalance request failed" - name: eth_estimateGas flow: @@ -375,6 +303,3 @@ scenarios: method: "eth_estimateGas" params: - from: "0x0" - - ensure: - check: "statusCode == 200" - message: "eth_estimateGas request failed" From abb72a02697ca2e8e75baf23ceff8c396f64cc75 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 16:23:46 -0600 Subject: [PATCH 23/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index edc2e4e2a5..8e37b52f4c 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/" + url: "/xxxx" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersion" # Deliberate typo to force failure + method: "web3_clientVersionxxx" # Deliberate typo to force failure params: [] - name: web3_sha3 From 6d69e1d15463337a7b21af9efe2a58383e3d6f79 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 16:32:00 -0600 Subject: [PATCH 24/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 8e37b52f4c..edc2e4e2a5 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/xxxx" + url: "/" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersionxxx" # Deliberate typo to force failure + method: "web3_clientVersion" # Deliberate typo to force failure params: [] - name: web3_sha3 From 46c03b2f915467061fa28a29128d4615cc66c97f Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 16:39:38 -0600 Subject: [PATCH 25/43] updated --- .github/actions/performance-tests/art.yaml | 75 ++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index edc2e4e2a5..9341c672c0 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -30,6 +30,9 @@ scenarios: jsonrpc: "2.0" method: "web3_clientVersion" # Deliberate typo to force failure params: [] + ensure: + - statusCode: 200 + - statusCode: 201 - name: web3_sha3 flow: @@ -41,6 +44,9 @@ scenarios: method: "web3_sha3" params: - "0x68656c6c6f20776f726c64" + ensure: + - statusCode: 200 + - statusCode: 201 - name: net_version flow: @@ -51,6 +57,9 @@ scenarios: jsonrpc: "2.0" method: "net_version" params: [] + ensure: + - statusCode: 200 + - statusCode: 201 - name: net_listening flow: @@ -61,6 +70,9 @@ scenarios: jsonrpc: "2.0" method: "net_listening" params: [] + ensure: + - statusCode: 200 + - statusCode: 201 - name: net_peerCount flow: @@ -71,6 +83,9 @@ scenarios: jsonrpc: "2.0" method: "net_peerCount" params: [] + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_chainId flow: @@ -81,6 +96,9 @@ scenarios: jsonrpc: "2.0" method: "eth_chainId" params: [] + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getStorageAt flow: @@ -92,6 +110,9 @@ scenarios: method: "eth_getStorageAt" params: - "latest" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getTransactionByBlockHashAndIndex flow: @@ -103,6 +124,9 @@ scenarios: method: "eth_getTransactionByBlockHashAndIndex" params: - "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getTransactionByBlockNumberAndIndex flow: @@ -115,6 +139,9 @@ scenarios: params: - "latest" - "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getUncleByBlockHashAndIndex flow: @@ -126,6 +153,9 @@ scenarios: method: "eth_getUncleByBlockHashAndIndex" params: - "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getUncleByBlockNumberAndIndex flow: @@ -138,6 +168,9 @@ scenarios: params: - "latest" - "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_newFilter flow: @@ -154,6 +187,9 @@ scenarios: address: "0x0", topics: [] } + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getFilterChanges flow: @@ -165,6 +201,9 @@ scenarios: method: "eth_getFilterChanges" params: - "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getFilterLogs flow: @@ -176,6 +215,9 @@ scenarios: method: "eth_getFilterLogs" params: - "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_blockNumber flow: @@ -186,6 +228,9 @@ scenarios: jsonrpc: "2.0" method: "eth_blockNumber" params: [] + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getBlockByHash flow: @@ -198,6 +243,9 @@ scenarios: params: - "0x0" - true + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getBlockByNumber flow: @@ -210,6 +258,9 @@ scenarios: params: - "latest" - true + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getBlockTransactionCountByHash flow: @@ -221,6 +272,9 @@ scenarios: method: "eth_getBlockTransactionCountByHash" params: - "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getBlockTransactionCountByNumber flow: @@ -232,6 +286,9 @@ scenarios: method: "eth_getBlockTransactionCountByNumber" params: - "latest" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getCode flow: @@ -244,6 +301,9 @@ scenarios: params: - "0x0" - "latest" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getTransactionByHash flow: @@ -255,6 +315,9 @@ scenarios: method: "eth_getTransactionByHash" params: - "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getTransactionReceipt flow: @@ -266,6 +329,9 @@ scenarios: method: "eth_getTransactionReceipt" params: - "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getLogs flow: @@ -280,6 +346,9 @@ scenarios: toBlock: "latest" address: "0x0" topics: ["0x0"] + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_getBalance flow: @@ -292,6 +361,9 @@ scenarios: params: - "0x0" - "latest" + ensure: + - statusCode: 200 + - statusCode: 201 - name: eth_estimateGas flow: @@ -303,3 +375,6 @@ scenarios: method: "eth_estimateGas" params: - from: "0x0" + ensure: + - statusCode: 200 + - statusCode: 201 From 60cdbe594c83dd91b109c20f8ffa13684b3a97da Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 16:44:00 -0600 Subject: [PATCH 26/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 9341c672c0..345dea327a 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/" + url: "/xxxx" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersion" # Deliberate typo to force failure + method: "web3_clientVersionxxxxx" # Deliberate typo to force failure params: [] ensure: - statusCode: 200 From 423bf1ec26fcfa10066e18bc37c44a7dc6e729cf Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 16:53:39 -0600 Subject: [PATCH 27/43] updated --- .github/workflows/ci-nightly-performance-testing.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 69ecde8b73..7611b10d14 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -17,9 +17,9 @@ jobs: with: node-version: 16 - - name: "START:LOCAL:NET" + - name: "START:LOCAL:NET:WITH:STATE" run: | - make start-e2e-test + make start-e2e-import-mainnet-test - name: "INSTALL:ARTILLERY" run: | From 4614aa4217a3371c31126ef038a08516cd44a4e6 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 17:23:19 -0600 Subject: [PATCH 28/43] updated --- .github/actions/performance-tests/art.yaml | 150 +++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 345dea327a..266e255154 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -30,9 +30,15 @@ scenarios: jsonrpc: "2.0" method: "web3_clientVersionxxxxx" # Deliberate typo to force failure params: [] + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: web3_sha3 flow: @@ -44,9 +50,15 @@ scenarios: method: "web3_sha3" params: - "0x68656c6c6f20776f726c64" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: net_version flow: @@ -57,9 +69,15 @@ scenarios: jsonrpc: "2.0" method: "net_version" params: [] + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: net_listening flow: @@ -70,9 +88,15 @@ scenarios: jsonrpc: "2.0" method: "net_listening" params: [] + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: net_peerCount flow: @@ -83,9 +107,15 @@ scenarios: jsonrpc: "2.0" method: "net_peerCount" params: [] + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_chainId flow: @@ -96,9 +126,15 @@ scenarios: jsonrpc: "2.0" method: "eth_chainId" params: [] + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getStorageAt flow: @@ -110,9 +146,15 @@ scenarios: method: "eth_getStorageAt" params: - "latest" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getTransactionByBlockHashAndIndex flow: @@ -124,9 +166,15 @@ scenarios: method: "eth_getTransactionByBlockHashAndIndex" params: - "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getTransactionByBlockNumberAndIndex flow: @@ -139,9 +187,15 @@ scenarios: params: - "latest" - "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getUncleByBlockHashAndIndex flow: @@ -153,9 +207,15 @@ scenarios: method: "eth_getUncleByBlockHashAndIndex" params: - "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getUncleByBlockNumberAndIndex flow: @@ -168,9 +228,15 @@ scenarios: params: - "latest" - "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_newFilter flow: @@ -187,9 +253,15 @@ scenarios: address: "0x0", topics: [] } + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getFilterChanges flow: @@ -201,9 +273,15 @@ scenarios: method: "eth_getFilterChanges" params: - "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getFilterLogs flow: @@ -215,9 +293,15 @@ scenarios: method: "eth_getFilterLogs" params: - "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_blockNumber flow: @@ -228,9 +312,15 @@ scenarios: jsonrpc: "2.0" method: "eth_blockNumber" params: [] + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getBlockByHash flow: @@ -243,9 +333,15 @@ scenarios: params: - "0x0" - true + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getBlockByNumber flow: @@ -258,9 +354,15 @@ scenarios: params: - "latest" - true + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getBlockTransactionCountByHash flow: @@ -272,9 +374,15 @@ scenarios: method: "eth_getBlockTransactionCountByHash" params: - "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getBlockTransactionCountByNumber flow: @@ -286,9 +394,15 @@ scenarios: method: "eth_getBlockTransactionCountByNumber" params: - "latest" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getCode flow: @@ -301,9 +415,15 @@ scenarios: params: - "0x0" - "latest" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getTransactionByHash flow: @@ -315,9 +435,15 @@ scenarios: method: "eth_getTransactionByHash" params: - "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getTransactionReceipt flow: @@ -329,9 +455,15 @@ scenarios: method: "eth_getTransactionReceipt" params: - "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getLogs flow: @@ -346,9 +478,15 @@ scenarios: toBlock: "latest" address: "0x0" topics: ["0x0"] + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_getBalance flow: @@ -361,9 +499,15 @@ scenarios: params: - "0x0" - "latest" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null - name: eth_estimateGas flow: @@ -375,6 +519,12 @@ scenarios: method: "eth_estimateGas" params: - from: "0x0" + capture: + - json: "$.error" + as: "error" ensure: - statusCode: 200 - statusCode: 201 + assert: + - subject: error + equals: null From 3e432631fdb270047098b0d16fa7c4ec318db004 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 17:36:45 -0600 Subject: [PATCH 29/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 266e255154..3046aa6e7e 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/xxxx" + url: "/" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersionxxxxx" # Deliberate typo to force failure + method: "web3_clientVersion" # Deliberate typo to force failure params: [] capture: - json: "$.error" From 537fc18b1c1f37a2eeafbad55ab1875ac7094384 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 17:49:34 -0600 Subject: [PATCH 30/43] updated --- .github/workflows/ci-nightly-performance-testing.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 7611b10d14..2e00911fb6 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -19,7 +19,8 @@ jobs: - name: "START:LOCAL:NET:WITH:STATE" run: | - make start-e2e-import-mainnet-test + make start-e2e-test + #make start-e2e-import-mainnet-test - name: "INSTALL:ARTILLERY" run: | From fb18c944ee4d1a05e47cf9cf9d9b6a660d93bc46 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 17:50:27 -0600 Subject: [PATCH 31/43] updated --- .github/actions/performance-tests/art.yaml | 154 ++++++++++----------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 3046aa6e7e..342b74f182 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,20 +24,20 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/" + url: "/xxxx" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersion" # Deliberate typo to force failure + method: "web3_clientVersionxxxxx" # Deliberate typo to force failure params: [] capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: web3_sha3 @@ -51,13 +51,13 @@ scenarios: params: - "0x68656c6c6f20776f726c64" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: net_version @@ -70,13 +70,13 @@ scenarios: method: "net_version" params: [] capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: net_listening @@ -89,13 +89,13 @@ scenarios: method: "net_listening" params: [] capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: net_peerCount @@ -108,13 +108,13 @@ scenarios: method: "net_peerCount" params: [] capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_chainId @@ -127,13 +127,13 @@ scenarios: method: "eth_chainId" params: [] capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getStorageAt @@ -147,13 +147,13 @@ scenarios: params: - "latest" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getTransactionByBlockHashAndIndex @@ -167,13 +167,13 @@ scenarios: params: - "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getTransactionByBlockNumberAndIndex @@ -188,13 +188,13 @@ scenarios: - "latest" - "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getUncleByBlockHashAndIndex @@ -208,13 +208,13 @@ scenarios: params: - "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getUncleByBlockNumberAndIndex @@ -229,13 +229,13 @@ scenarios: - "latest" - "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_newFilter @@ -254,13 +254,13 @@ scenarios: topics: [] } capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getFilterChanges @@ -274,13 +274,13 @@ scenarios: params: - "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getFilterLogs @@ -294,13 +294,13 @@ scenarios: params: - "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_blockNumber @@ -313,13 +313,13 @@ scenarios: method: "eth_blockNumber" params: [] capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getBlockByHash @@ -334,13 +334,13 @@ scenarios: - "0x0" - true capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getBlockByNumber @@ -355,13 +355,13 @@ scenarios: - "latest" - true capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getBlockTransactionCountByHash @@ -375,13 +375,13 @@ scenarios: params: - "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getBlockTransactionCountByNumber @@ -395,13 +395,13 @@ scenarios: params: - "latest" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getCode @@ -416,13 +416,13 @@ scenarios: - "0x0" - "latest" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getTransactionByHash @@ -436,13 +436,13 @@ scenarios: params: - "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getTransactionReceipt @@ -456,13 +456,13 @@ scenarios: params: - "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getLogs @@ -479,13 +479,13 @@ scenarios: address: "0x0" topics: ["0x0"] capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_getBalance @@ -500,13 +500,13 @@ scenarios: - "0x0" - "latest" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null - name: eth_estimateGas @@ -520,11 +520,11 @@ scenarios: params: - from: "0x0" capture: - - json: "$.error" - as: "error" + - json: "$" + as: "response" ensure: - statusCode: 200 - statusCode: 201 assert: - - subject: error + - subject: response.error equals: null From 1d9c56dff78eb47f76fd511247a63e95f8cc8177 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 17:55:54 -0600 Subject: [PATCH 32/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 342b74f182..4a65c91e10 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/xxxx" + url: "/" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersionxxxxx" # Deliberate typo to force failure + method: "web3_clientVersion" # Deliberate typo to force failure params: [] capture: - json: "$" From 63da90b98e40d52326539b22d144ad585d78d85a Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 18:00:54 -0600 Subject: [PATCH 33/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- .github/workflows/ci-nightly-performance-testing.yaml | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 4a65c91e10..b63b3c3f63 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/" + url: "/xxx" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersion" # Deliberate typo to force failure + method: "web3_clientVersionxxxx" # Deliberate typo to force failure params: [] capture: - json: "$" diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 2e00911fb6..7611b10d14 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -19,8 +19,7 @@ jobs: - name: "START:LOCAL:NET:WITH:STATE" run: | - make start-e2e-test - #make start-e2e-import-mainnet-test + make start-e2e-import-mainnet-test - name: "INSTALL:ARTILLERY" run: | From 2d95be4e1d847d5ac4906395ef512531e33b3d5b Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 18:01:35 -0600 Subject: [PATCH 34/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index b63b3c3f63..4a65c91e10 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/xxx" + url: "/" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersionxxxx" # Deliberate typo to force failure + method: "web3_clientVersion" # Deliberate typo to force failure params: [] capture: - json: "$" From 22e851c9fe8ba664dee69340c3601f8f6f072fc1 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Fri, 28 Jun 2024 18:21:00 -0600 Subject: [PATCH 35/43] updated chagne log --- changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/changelog.md b/changelog.md index a16cd90d2d..867570a010 100644 --- a/changelog.md +++ b/changelog.md @@ -98,6 +98,7 @@ * [2192](https://github.com/zeta-chain/node/pull/2192) - Added release status checker and updater pipeline that will update release statuses when they go live on network * [2335](https://github.com/zeta-chain/node/pull/2335) - ci: updated the artillery report to publish to artillery cloud * [2377](https://github.com/zeta-chain/node/pull/2377) - ci: adjusted sast-linters.yml to not scan itself, nor alert on removal of nosec. +* [2400](https://github.com/zeta-chain/node/pull/2400) - ci: adjusted the performance test to pass or fail pipeline based on test results, alert slack, and launch network with state. Fixed connection issues as well. ### Documentation From 7fe8dc29288fcc2471d942e8096bc578ae81eeca Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Sun, 30 Jun 2024 18:02:21 -0600 Subject: [PATCH 36/43] removed comment --- .github/actions/performance-tests/art.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 4a65c91e10..70d2ba1f85 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -28,7 +28,7 @@ scenarios: json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersion" # Deliberate typo to force failure + method: "web3_clientVersion" params: [] capture: - json: "$" From 11456e9f18da53f4edb83c4e81adbf98388f8d14 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Mon, 1 Jul 2024 10:09:49 -0600 Subject: [PATCH 37/43] updated --- .github/workflows/ci-nightly-performance-testing.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 7611b10d14..d0c291387a 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -59,7 +59,7 @@ jobs: path: ./artillery_report.html - name: "NOTIFY:SLACK:FAILURE" - if: failure() + if: failure() || github.event_name == 'schedule' uses: 8398a7/action-slack@v3 with: status: ${{ job.status }} From 414a3bb5e3837a9e963672efd7061fc74c3625dd Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Mon, 1 Jul 2024 10:11:56 -0600 Subject: [PATCH 38/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 70d2ba1f85..4b6379bea5 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/" + url: "/xxx" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersion" + method: "web3_clientVersionxxxx" params: [] capture: - json: "$" From 2bc68ebbfe2620c2cd78734fc163c7a0c0a57201 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Mon, 1 Jul 2024 10:12:37 -0600 Subject: [PATCH 39/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 4b6379bea5..70d2ba1f85 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/xxx" + url: "/" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersionxxxx" + method: "web3_clientVersion" params: [] capture: - json: "$" From c901dafb968826ddf5da4973fd5e2c1708248e47 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Mon, 1 Jul 2024 10:34:45 -0600 Subject: [PATCH 40/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- .github/workflows/ci-nightly-performance-testing.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 70d2ba1f85..185b2565ad 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/" + url: "/xxxx" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersion" + method: "web3_clientVersionxxx" params: [] capture: - json: "$" diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index d0c291387a..c4a9a19bf6 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -59,7 +59,7 @@ jobs: path: ./artillery_report.html - name: "NOTIFY:SLACK:FAILURE" - if: failure() || github.event_name == 'schedule' + #if: failure() && github.event_name == 'schedule' uses: 8398a7/action-slack@v3 with: status: ${{ job.status }} From b8ea1bff4af54c860ac3e51016f9463dda74be78 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Mon, 1 Jul 2024 11:06:30 -0600 Subject: [PATCH 41/43] updated --- .github/workflows/ci-nightly-performance-testing.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index c4a9a19bf6..4c1dd8f6a0 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -60,6 +60,7 @@ jobs: - name: "NOTIFY:SLACK:FAILURE" #if: failure() && github.event_name == 'schedule' + if: failure() uses: 8398a7/action-slack@v3 with: status: ${{ job.status }} From f2948b5a1fc4c5604c7de4c4998522f5893a1d41 Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Mon, 1 Jul 2024 15:31:17 -0600 Subject: [PATCH 42/43] updated --- .github/workflows/ci-nightly-performance-testing.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 4c1dd8f6a0..a6a566f4aa 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -59,8 +59,7 @@ jobs: path: ./artillery_report.html - name: "NOTIFY:SLACK:FAILURE" - #if: failure() && github.event_name == 'schedule' - if: failure() + if: failure() && github.event_name == 'schedule' uses: 8398a7/action-slack@v3 with: status: ${{ job.status }} From dd2b827e12a2c394b0d60e77c844d36dcce2a40c Mon Sep 17 00:00:00 2001 From: Grant Zukel Date: Mon, 1 Jul 2024 15:45:46 -0600 Subject: [PATCH 43/43] updated --- .github/actions/performance-tests/art.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 185b2565ad..70d2ba1f85 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -24,11 +24,11 @@ scenarios: - name: web3_clientVersion flow: - post: - url: "/xxxx" + url: "/" json: id: 1 jsonrpc: "2.0" - method: "web3_clientVersionxxx" + method: "web3_clientVersion" params: [] capture: - json: "$"