Add REST API query normalization (#9591)

Adds a means to normalize REST API queries. --------- Signed-off-by: Edwin Greene <[email protected]>
hashgraph · Oct 29, 2024 · 732ef35 · 732ef35
1 parent e51c3b2
commit 732ef35
Show file tree

Hide file tree

Showing 5 changed files with 328 additions and 11 deletions.
diff --git a/hedera-mirror-rest/__tests__/middleware/requestNormalizer.test.js b/hedera-mirror-rest/__tests__/middleware/requestNormalizer.test.js
@@ -0,0 +1,161 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {normalizeRequestQueryParams} from '../../middleware/requestNormalizer.js';
+import request from 'supertest';
+import {openApiValidator} from '../../middleware/index.js';
+import express from 'express';
+
+const app = express();
+openApiValidator(app);
+
+describe('request normalizer', () => {
+  const testSpecs = [
+    {
+      description: 'Unknown path',
+      input: '/api/v1/unknown/123?limit=2',
+      expected: '/api/v1/unknown/123?limit=2',
+    },
+    {
+      description: 'Block query parameters are added',
+      input: '/api/v1/blocks',
+      expected: '/api/v1/blocks?limit=25&order=desc',
+    },
+    {
+      description: 'Non sortable parameters are not sorted',
+      input: '/api/v1/blocks?order=asc&block.number=3&block.number=2',
+      expected: '/api/v1/blocks?block.number=3&block.number=2&limit=25&order=asc',
+    },
+    {
+      description: 'Account parameters are added',
+      input: '/api/v1/accounts',
+      expected: '/api/v1/accounts?balance=true&limit=25&order=asc',
+    },
+    {
+      description: 'Eq and ne parameters',
+      input: '/api/v1/accounts?account.id=eq:1001&account.id=ne:3',
+      expected: '/api/v1/accounts?account.id=eq:1001&account.id=ne:3&balance=true&limit=25&order=asc',
+    },
+    {
+      description: 'Default query parameters are added to existing query parameters and sorted',
+      input: '/api/v1/accounts?limit=3&account.id=gt:0.0.20&account.id=lt:0.0.21',
+      expected: '/api/v1/accounts?account.id=gt:0.0.20&account.id=lt:0.0.21&balance=true&limit=3&order=asc',
+    },
+    {
+      description: 'Multiple instances of the same parameter are allowed',
+      input: '/api/v1/accounts?limit=2&limit=3&limit=5',
+      expected: '/api/v1/accounts?balance=true&limit=2&limit=3&limit=5&order=asc',
+    },
+    {
+      description: 'Accounts with path parameter has default parameter added',
+      input: '/api/v1/accounts/0.0.1001/nfts',
+      expected: '/api/v1/accounts/0.0.1001/nfts?limit=25&order=desc',
+    },
+    {
+      description:
+        'Accounts with path parameter and query parameters have default parameter added and parameters sorted',
+      input: '/api/v1/accounts/0.0.1001/nfts?serialnumber=gte:2&spender.id=gte:2004&token.id=gte:1500&order=asc',
+      expected:
+        '/api/v1/accounts/0.0.1001/nfts?limit=25&order=asc&serialnumber=gte:2&spender.id=gte:2004&token.id=gte:1500',
+    },
+    {
+      description: 'Contract result log parameters are added and sorted',
+      input: '/api/v1/contracts/results/logs?index=lt:1&timestamp=1639010141.000000000&topic0=A',
+      expected: '/api/v1/contracts/results/logs?index=lt:1&limit=25&order=desc&timestamp=1639010141.000000000&topic0=A',
+    },
+    {
+      description: 'No parameters are added',
+      input: '/api/v1/contracts/62cf9068fed962cf9aaabbb962cf9068fed9dddd',
+      expected: '/api/v1/contracts/62cf9068fed962cf9aaabbb962cf9068fed9dddd',
+    },
+    {
+      description:
+        'Contracts with path parameter and query parameter have default parameters added and parameters sorted',
+      input: '/api/v1/contracts/62cf9068fed962cf9aaabbb962cf9068fed9dddd/results?limit=3',
+      expected: '/api/v1/contracts/62cf9068fed962cf9aaabbb962cf9068fed9dddd/results?internal=false&limit=3&order=desc',
+    },
+    {
+      description: 'Accounts nfts with path parameter and query parameters',
+      input: '/api/v1/accounts/0.0.1001/nfts?token.id=gte:1500&serialnumber=gte:2&spender.id=gte:2004&order=asc',
+      expected:
+        '/api/v1/accounts/0.0.1001/nfts?limit=25&order=asc&serialnumber=gte:2&spender.id=gte:2004&token.id=gte:1500',
+    },
+    {
+      description: 'Token nfts with path parameter',
+      input: '/api/v1/tokens/1500/nfts/2/transactions',
+      expected: '/api/v1/tokens/1500/nfts/2/transactions?limit=25&order=desc',
+    },
+    {
+      description: 'Token nfts with shard realm num path parameter',
+      input: '/api/v1/tokens/0.0.1500/nfts/2/transactions',
+      expected: '/api/v1/tokens/0.0.1500/nfts/2/transactions?limit=25&order=desc',
+    },
+    {
+      description: 'Token nfts with path parameter and query parameters',
+      input: '/api/v1/tokens/1500/nfts/2/transactions?timestamp=gte:1234567890.000000005&order=asc',
+      expected: '/api/v1/tokens/1500/nfts/2/transactions?limit=25&order=asc&timestamp=gte:1234567890.000000005',
+    },
+    {
+      description: 'Accounts nfts with shard realm num path parameter and query parameters',
+      input: '/api/v1/accounts/0.0.1001/nfts?token.id=gte:1500&serialnumber=gte:2&spender.id=gte:2004',
+      expected:
+        '/api/v1/accounts/0.0.1001/nfts?limit=25&order=desc&serialnumber=gte:2&spender.id=gte:2004&token.id=gte:1500',
+    },
+    {
+      description: 'Topics messages with path parameter',
+      input: '/api/v1/topics/7/messages',
+      expected: '/api/v1/topics/7/messages?limit=25&order=asc',
+    },
+    {
+      description: 'Balance is collapsed to the last value',
+      input: '/api/v1/accounts?limit=3&balance=true&balance=false',
+      expected: '/api/v1/accounts?balance=false&limit=3&order=asc',
+    },
+    {
+      description: 'Two collapsable params are collapsed',
+      input:
+        '/api/v1/transactions/0xae8bebf1c9fa0f309356e48057f6047af7cde63037d0509d16ddc3b20e085158bfdf14d15345c1b18b199b72fed4ac6f?scheduled=false&scheduled=true&nonce=2&nonce=1',
+      expected:
+        '/api/v1/transactions/0xae8bebf1c9fa0f309356e48057f6047af7cde63037d0509d16ddc3b20e085158bfdf14d15345c1b18b199b72fed4ac6f?nonce=1&scheduled=true',
+    },
+    {
+      description: 'Timestamp values are not sorted',
+      input: '/api/v1/transactions?timestamp=1639010141.000000001&timestamp=1639010141.000000000',
+      expected:
+        '/api/v1/transactions?limit=25&order=desc&timestamp=1639010141.000000001&timestamp=1639010141.000000000',
+    },
+  ];
+
+  const setupRoute = (spec) => {
+    const route = spec.input.split('?')[0];
+    app.get(route, (req, res) => {
+      const actual = normalizeRequestQueryParams(req.openapi?.openApiRoute, req.path, req.query);
+      res.set('actual', actual);
+      res.sendStatus(200);
+    });
+  };
+
+  testSpecs.forEach((spec) => {
+    setupRoute(spec);
+    test(spec.description, async () => {
+      await request(app)
+        .get(spec.input)
+        .then((res) => {
+          expect(res.get('actual')).toEqual(spec.expected);
+        });
+    });
+  });
+});
diff --git a/hedera-mirror-rest/api/v1/openapi.yml b/hedera-mirror-rest/api/v1/openapi.yml
@@ -1049,7 +1049,7 @@ paths:
           schema:
             type: string
         - $ref: "#/components/parameters/limitQueryParam"
-        - $ref: "#/components/parameters/orderQueryParamDesc"
+        - $ref: "#/components/parameters/orderQueryParam"
         - name: sequencenumber
           in: query
           example: 2
@@ -1284,7 +1284,7 @@ paths:
         - $ref: "#/components/parameters/accountIdQueryParam"
         - $ref: "#/components/parameters/accountPublicKeyQueryParam"
         - $ref: "#/components/parameters/limitQueryParam"
-        - $ref: "#/components/parameters/orderQueryParam"
+        - $ref: "#/components/parameters/orderQueryParamDesc"
         - $ref: "#/components/parameters/timestampQueryParam"
       responses:
         200:
@@ -4192,7 +4192,9 @@ components:
     balanceQueryParam:
       name: balance
       in: query
-      description: Whether to include balance information or not. If included, token balances are limited to at most 50 per account as outlined in HIP-367.
+      description: >-
+        Whether to include balance information or not. If included, token balances are limited to at most 50 per
+        account as outlined in HIP-367. If multiple values are provided the last value will be the only value used.
       example: true
       schema:
         type: boolean
@@ -4431,7 +4433,8 @@ components:
       description: >-
         Filter the query result by the nonce of the transaction. A zero nonce represents user submitted transactions
         while a non-zero nonce is generated by main nodes. The filter honors the last value. If not specified,
-        all transactions with specified payer account ID and valid start timestamp match.
+        all transactions with specified payer account ID and valid start timestamp match. If multiple values are
+        provided the last value will be the only value used.
       example: 0
       schema:
         format: int32
@@ -4521,7 +4524,7 @@ components:
       description: >-
         Filter transactions by the scheduled flag. If true, return information for the scheduled transaction. If false,
         return information for the non-scheduled transaction. If not present, return information for all transactions
-        matching transactionId.
+        matching transactionId. If multiple values are provided the last value will be the only value used.
       schema:
         type: boolean
     scheduledQueryParamWithDefault:
@@ -5076,7 +5079,7 @@ components:
     blockHashQueryParam:
       name: block.hash
       in: query
-      description: The block's hash
+      description: The block's hash. If multiple values are provided the last value will be the only value used.
       examples:
         noValue:
           summary: --

diff --git a/hedera-mirror-rest/middleware/openapiHandler.js b/hedera-mirror-rest/middleware/openapiHandler.js
@@ -23,9 +23,13 @@ import swaggerUi from 'swagger-ui-express';
 
 // files
 import config from '../config';
+import {isTestEnv} from '../utils.js';
 
 let v1OpenApiDocument;
 let v1OpenApiFile;
+let openApiMap;
+
+const OPEN_API_PARAMETER_LOCATION = '#/components/parameters/';
 
 /**
  * Check if apiVersion is currently supported
@@ -80,6 +84,58 @@ const getV1OpenApiObject = () => {
   return v1OpenApiDocument;
 };
 
+/**
+ * Get the path to parameter properties map for the OpenApi Spec
+ *
+ * @returns {Map<string, Array<{parameterName, defaultValue}>>}
+ */
+const getOpenApiMap = () => {
+  if (_.isUndefined(openApiMap)) {
+    const openApiObject = getV1OpenApiObject();
+    const map = new Map();
+    Object.keys(openApiObject.paths).forEach((path) => {
+      const parameters = getOpenApiParameters(path, openApiObject);
+      map.set(path, parameters);
+    });
+    openApiMap = map;
+  }
+
+  return openApiMap;
+};
+
+/**
+ * Given a path, gets the query parameters and their default values
+ * @param path {string}
+ * @param openApiObject
+ * @returns {Array<{parameterName, defaultValue}>}
+ */
+const getOpenApiParameters = (path, openApiObject) => {
+  const pathObject = openApiObject.paths[path];
+  const parameters = pathObject?.get?.parameters;
+  if (parameters === undefined) {
+    return {};
+  }
+
+  return (
+    parameters
+      // Each open api parameter is prefixed by #/components/parameters/
+      .filter((p) => p.$ref?.includes(OPEN_API_PARAMETER_LOCATION))
+      .map((p) => p.$ref.substring(OPEN_API_PARAMETER_LOCATION.length))
+      .map((p) => openApiObject.components.parameters[p])
+      .filter((p) => p.in !== 'path')
+      .map((p) => {
+        const parameterName = p.name;
+        let defaultValue = p.schema?.default;
+        if (defaultValue !== undefined && !_.isString(defaultValue)) {
+          // Convert all values to strings
+          defaultValue = '' + defaultValue;
+        }
+
+        return {parameterName, defaultValue};
+      })
+  );
+};
+
 const serveSpec = (req, res) => res.type('text/yaml').send(getV1OpenApiFile());
 
 /**
@@ -109,10 +165,10 @@ const openApiValidator = (app) => {
       ignoreUndocumented: true,
       validateRequests: false,
       validateResponses: {
-        allErrors: true,
+        allErrors: isTestEnv(),
       },
     })
   );
 };
 
-export {getV1OpenApiObject, openApiValidator, serveSwaggerDocs};
+export {getV1OpenApiObject, getOpenApiMap, openApiValidator, serveSwaggerDocs};