Merge branch 'dev' into fix/remove-header-based-logger

PerimeterX · Jan 17, 2024 · c96f1fe · c96f1fe
2 parents c36a2d0 + 69478df
commit c96f1fe
Show file tree

Hide file tree

Showing 11 changed files with 79 additions and 29 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [3.14.0] - 2024-01-11
+
+### Added
+- Support for url decode reserved characters feature
+
 ## [3.13.0] - 2023-12-21
 
 ### Added

diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@
 [PerimeterX](http://www.perimeterx.com) Shared base for NodeJS enforcers
 =============================================================
 
-> Latest stable version: [v3.13.0](https://www.npmjs.com/package/perimeterx-node-core)
+> Latest stable version: [v3.14.0](https://www.npmjs.com/package/perimeterx-node-core)
 
 This is a shared base implementation for PerimeterX Express enforcer and future NodeJS enforcers. For a fully functioning implementation example, see the [Node-Express enforcer](https://github.com/PerimeterX/perimeterx-node-express/) implementation.
 

diff --git a/lib/pxapi.js b/lib/pxapi.js
@@ -75,6 +75,9 @@ function buildRequestData(ctx, config) {
     if (ctx.serverInfoRegion) {
         data.additional['server_info_region'] = ctx.serverInfoRegion;
     }
+    if (ctx.isRawUrlDifferentFromNormalizedUrl) {
+        data.additional['raw_url'] = ctx.rawUrl;
+    }
 
     if (ctx.additionalFields && ctx.additionalFields.loginCredentials) {
         const { loginCredentials } = ctx.additionalFields;

diff --git a/lib/pxconfig.js b/lib/pxconfig.js
@@ -103,7 +103,8 @@ class PxConfig {
             ['JWT_HEADER_USER_ID_FIELD_NAME', 'px_jwt_header_user_id_field_name'],
             ['JWT_HEADER_ADDITIONAL_FIELD_NAMES', 'px_jwt_header_additional_field_names'],
             ['CUSTOM_IS_SENSITIVE_REQUEST', 'px_custom_is_sensitive_request'],
-            ['FIRST_PARTY_TIMEOUT_MS', 'px_first_party_timeout_ms']
+            ['FIRST_PARTY_TIMEOUT_MS', 'px_first_party_timeout_ms'],
+            ['URL_DECODE_RESERVED_CHARACTERS', 'px_url_decode_reserved_characters']
         ];
 
         configKeyMapping.forEach(([targetKey, sourceKey]) => {
@@ -363,7 +364,8 @@ function pxDefaultConfig() {
         JWT_HEADER_USER_ID_FIELD_NAME: '',
         JWT_HEADER_ADDITIONAL_FIELD_NAMES: [],
         CUSTOM_IS_SENSITIVE_REQUEST: '',
-        FIRST_PARTY_TIMEOUT_MS: 4000
+        FIRST_PARTY_TIMEOUT_MS: 4000,
+        URL_DECODE_RESERVED_CHARACTERS: false
     };
 }
 

diff --git a/lib/pxcontext.js b/lib/pxcontext.js
@@ -20,7 +20,9 @@ class PxContext {
         this.hostname = req.hostname || req.get('host');
         this.userAgent = userAgent;
         this.uri = req.originalUrl || '/';
-        this.fullUrl = req.protocol + '://' + req.get('host') + req.originalUrl;
+        this.rawUrl = req.originalFullUrl;
+        this.isRawUrlDifferentFromNormalizedUrl = req.originalFullUrl !== req.requestNormalizedUrl;
+        this.fullUrl = req.requestNormalizedUrl;
         this.originalRequest = req.originalRequest || req;
         this.httpVersion = req.httpVersion || '';
         this.httpMethod = req.method || '';

diff --git a/lib/pxenforcer.js b/lib/pxenforcer.js
@@ -61,32 +61,36 @@ class PxEnforcer {
     }
 
     enforce(req, res, cb) {
-        const requestUrl = req.originalUrl;
+        if (!this._config.ENABLE_MODULE) {
+            this.logger.debug('Request will not be verified, module is disabled');
+            return cb();
+        }
+        const fullUrlFromRequest = pxUtil.getFullUrlFromRequest(req);
+        req.originalFullUrl = req.originalFullUrl || fullUrlFromRequest;
+        const normalizedUrl = this._normalizeUrl(fullUrlFromRequest);
+
+        req['requestNormalizedUrl'] = normalizedUrl.href;
+
         const userAgent = req.get('user-agent') || '';
         const ipAddress = pxUtil.extractIP(this._config, req);
 
         this.logger.debug('Starting request verification');
 
-        if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_VENDOR_PATH}`)) {
+        if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_VENDOR_PATH}`)) {
             // reverse proxy client
             return pxProxy.getClient(req, this._config, ipAddress, cb);
         }
 
-        if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_XHR_PATH}`)) {
+        if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_XHR_PATH}`)) {
             //reverse proxy xhr
             return pxProxy.sendXHR(req, this._config, ipAddress, this.reversePrefix, cb);
         }
 
-        if (requestUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_CAPTCHA_PATH}`)) {
+        if (req.originalUrl.startsWith(`/${this.reversePrefix}${this._config.FIRST_PARTY_CAPTCHA_PATH}`)) {
             // reverse proxy captcha
             return pxProxy.getCaptcha(req, this._config, ipAddress, this.reversePrefix, cb);
         }
 
-        if (!this._config.ENABLE_MODULE) {
-            this.logger.debug('Request will not be verified, module is disabled');
-            return cb();
-        }
-
         if (this._config.FILTER_BY_METHOD.includes(req.method.toUpperCase())) {
             this.logger.debug(`Skipping verification for filtered method ${req.method}`);
             return cb();
@@ -150,6 +154,19 @@ class PxEnforcer {
         }
     }
 
+    _normalizeUrl(originalUrl) {
+        let normalizedUrl = new URL(originalUrl);
+        if (this._config.URL_DECODE_RESERVED_CHARACTERS) {
+            try {
+                normalizedUrl = new URL(`${normalizedUrl.origin}${decodeURIComponent(normalizedUrl.pathname)}${normalizedUrl.search}`);
+            } catch (e) {
+                this.logger.debug(`unable to URL decode reserved characters: ${e}`);
+            }
+        }
+        normalizedUrl.pathname = normalizedUrl.pathname.replace(/\/+$/, '').replace(/\/+/g, '/');
+        return normalizedUrl;
+    }
+
     _tryModifyContext(ctx, req) {
         if (this._config.MODIFY_CONTEXT && typeof this._config.MODIFY_CONTEXT === 'function') {
             try {
@@ -406,7 +423,7 @@ class PxEnforcer {
     }
 
     getActivityDetails(ctx) {
-        return {
+        const details = {
             client_uuid: ctx.uuid,
             http_version: ctx.httpVersion,
             risk_rtt: ctx.riskRtt,
@@ -416,6 +433,11 @@ class PxEnforcer {
             request_cookie_names: ctx.requestCookieNames,
             enforcer_start_time: ctx.enforcerStartTime,
         };
+
+        if (ctx.isRawUrlDifferentFromNormalizedUrl) {
+            details.raw_url = ctx.rawUrl;
+        }
+        return details;
     }
 
     /**

diff --git a/lib/pxutil.js b/lib/pxutil.js
@@ -19,6 +19,11 @@ const { EMAIL_ADDRESS_REGEX, HASH_ALGORITHM } = require('./utils/constants');
  * @param {Object} headers - request headers in key value format.
  * @return {Array} request headers an array format.
  */
+
+function getFullUrlFromRequest(req) {
+    return `${req.protocol}://${req.get('host')}${req.originalUrl}`;
+}
+
 function formatHeaders(headers, sensitiveHeaders) {
     const retval = [];
     try {
@@ -407,5 +412,6 @@ module.exports = {
     isEmailAddress,
     isGraphql,
     tryOrNull,
-    appendContentType
+    appendContentType,
+    getFullUrlFromRequest
 };
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "perimeterx-node-core",
-    "version": "3.13.0",
+    "version": "3.14.0",
     "description": "PerimeterX NodeJS shared core for various applications to monitor and block traffic according to PerimeterX risk score",
     "main": "index.js",
     "scripts": {

diff --git a/test/pxcors.test.js b/test/pxcors.test.js
@@ -37,7 +37,11 @@ describe('PX Cors - pxCors.js', () => {
         req.protocol = 'http';
         req.ip = '1.2.3.4';
         req.hostname = 'example.com';
+        req.host = 'example.com';
         req.get = (key) => {
+            if (key === 'host') {
+                return req.host;
+            }
             return req.headers[key] || '';
         };
 

diff --git a/test/pxenforcer.test.js b/test/pxenforcer.test.js
@@ -40,6 +40,9 @@ describe('PX Enforcer - pxenforcer.js', () => {
         req.ip = '1.2.3.4';
         req.hostname = 'example.com';
         req.get = (key) => {
+            if (key === 'host') {
+                return req.host;
+            }
             return req.headers[key] || '';
         };