diff --git a/docs/database/bootstrap.md b/docs/database/bootstrap.md index f3eabde3b45..5e0b49048b8 100644 --- a/docs/database/bootstrap.md +++ b/docs/database/bootstrap.md @@ -7,6 +7,7 @@ This guide provides step-by-step instructions for setting up a fresh PostgreSQL ## Table of Contents - [Prerequisites](#prerequisites) + - [1. Optional High-Performance Decompressors](#1-optional-high-performance-decompressors) - [Database Initialization and Data Import](#database-initialization-and-data-import) - [1. Download the Required Scripts and Configuration File](#1-download-the-required-scripts-and-configuration-file) - [2. Edit the `bootstrap.env` Configuration File](#2-edit-the-bootstrapenv-configuration-file) @@ -15,9 +16,15 @@ This guide provides step-by-step instructions for setting up a fresh PostgreSQL - [3.2. List Available Versions](#32-list-available-versions) - [3.3. Select a Version](#33-select-a-version) - [3.4. Download the Data](#34-download-the-data) + - [Download Minimal DB Data Files](#download-minimal-db-data-files) + - [Download Full DB Data Files](#download-full-db-data-files) - [4. Check Version Compatibility](#4-check-version-compatibility) - [5. Run the Bootstrap Script](#5-run-the-bootstrap-script) - [6. Monitoring and Managing the Import Process](#6-monitoring-and-managing-the-import-process) + - [6.1. Monitoring the Import Process](#61-monitoring-the-import-process) + - [6.2. Stopping the Script](#62-stopping-the-script) + - [6.3. Resuming the Import Process](#63-resuming-the-import-process) + - [6.4. Start the Mirrornode Importer](#64-start-the-mirrornode-importer) - [Handling Failed Imports](#handling-failed-imports) - [Additional Notes](#additional-notes) - [Troubleshooting](#troubleshooting) @@ -37,6 +44,16 @@ This guide provides step-by-step instructions for setting up a fresh PostgreSQL - `realpath` - `flock` - `curl` + - `b3sum` + + ### 1. Optional High-Performance Decompressors + + The script automatically detects and uses faster alternatives to `gunzip` if they are available in the system's or user's PATH: + + - [rapidgzip](https://github.com/mxmlnkn/rapidgzip) - A high-performance parallel gzip decompressor (fastest option, even for single-threaded decompression) + - [igzip](https://github.com/intel/isa-l) - Intel's optimized gzip implementation from ISA-L (second fastest option) + + These tools can significantly improve decompression performance during the import process. If neither is available, the script will fall back to using standard `gunzip`. 4. Install the [Google Cloud SDK](https://cloud.google.com/sdk/docs/install), then authenticate: @@ -144,7 +161,7 @@ gcloud config set project YOUR_GCP_PROJECT_ID To see the available versions of the database export, list the contents of the bucket: ```bash -gsutil -m ls gs://mirrornode-db-export/ +gcloud storage ls gs://mirrornode-db-export/ ``` This will display the available version directories. @@ -162,13 +179,32 @@ This will display the available version directories. #### 3.4. Download the Data -Create a directory to store the data and download all files and subdirectories for the selected version: +Choose one of the following download options based on your needs: + +##### Download Minimal DB Data Files + +Create a directory and download only the minimal database files: + +```bash +mkdir -p /path/to/db_export +export CLOUDSDK_STORAGE_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS=1 && \ +VERSION_NUMBER= && \ +gcloud storage rsync -r -x '.*_part_\d+_\d+_\d+_atma\.csv\.gz$' "gs://mirrornode-db-export/$VERSION_NUMBER/" /path/to/db_export/ +``` + +##### Download Full DB Data Files + +Create a directory and download all files and subdirectories for the selected version: ```bash mkdir -p /path/to/db_export -gsutil -m cp -r gs://mirrornode-db-export//* /path/to/db_export/ +export CLOUDSDK_STORAGE_SLICED_OBJECT_DOWNLOAD_MAX_COMPONENTS=1 && \ +VERSION_NUMBER= && \ +gcloud storage rsync -r "gs://mirrornode-db-export/$VERSION_NUMBER/" /path/to/db_export/ ``` +For both options: + - Replace `/path/to/db_export` with your desired directory path. - Replace `` with the version you selected (e.g., `0.111.0`). - Ensure all files and subdirectories are downloaded into this single parent directory. @@ -207,18 +243,30 @@ The `bootstrap.sh` script initializes the database and imports the data. It is d # Should list bootstrap.sh and bootstrap.env ``` -2. **Run the Bootstrap Script Using `nohup` and Redirect Output to `bootstrap.log`:** +2. **Run the Bootstrap Script Using `setsid` and Redirect Output to `bootstrap.log`:** + + To ensure the script continues running even if your SSH session is terminated, run it in a new session using `setsid`. The script handles its own logging, but we redirect stderr to capture any startup errors: + + For a minimal database import (default): + + ```bash + setsid ./bootstrap.sh 8 /path/to/db_export > /dev/null 2>> bootstrap.log & + ``` - To ensure the script continues running even if your SSH session is terminated, run it using `nohup`, redirect stdout and stderr to `bootstrap.log`, and save its process ID (PID) to a file. + For a full database import: ```bash - nohup setsid ./bootstrap.sh 8 /path/to/db_export > /dev/null 2>> bootstrap.log & + setsid ./bootstrap.sh 8 --full /path/to/db_export > /dev/null 2>> bootstrap.log & ``` - - The script handles logging internally to `bootstrap.log`, and the execution command will also append stdout/stderr of the script itself to the log file. + - The script handles logging internally to `bootstrap.log`, and the execution command will also append stderr to the log file - `8` refers to the number of CPU cores to use for parallel processing. Adjust this number based on your system's resources. - `/path/to/db_export` is the directory where you downloaded the database export data. - - `bootstrap.pid` stores the PID of the running script for later use. + - The script creates several tracking files: + + - `bootstrap.pid` stores the process ID used for cleanup of all child processes if interrupted + - `bootstrap_tracking.txt` tracks the progress of each file's import and hash verification + - `bootstrap_discrepancies.log` records any data verification issues - **Important**: The SKIP_DB_INIT flag file is automatically created by the script after a successful database initialization. Do not manually create or delete this file. If you need to force the script to reinitialize the database in future runs, remove the flag file using: @@ -240,7 +288,7 @@ The `bootstrap.sh` script initializes the database and imports the data. It is d ### 6. Monitoring and Managing the Import Process -#### **Monitoring the Import Process:** +#### **6.1. Monitoring the Import Process:** - **Check the Log File:** @@ -258,8 +306,22 @@ The `bootstrap.sh` script initializes the database and imports the data. It is d ``` - This file tracks the status of each file being imported. + - Each line contains the file name, followed by two status indicators: + + Import Status: + + - `NOT_STARTED`: File has not begun importing + - `IN_PROGRESS`: File is currently being imported + - `IMPORTED`: File was successfully imported + - `FAILED_TO_IMPORT`: File import failed + + Hash Verification Status: + + - `HASH_UNVERIFIED`: BLAKE3 hash has not been verified yet + - `HASH_VERIFIED`: BLAKE3 hash verification passed + - `HASH_FAILED`: BLAKE3 hash verification failed -#### **Stopping the Script** +#### **6.2. Stopping the Script** If you need to stop the script before it completes: @@ -283,17 +345,18 @@ If you need to stop the script before it completes: **Note:** Ensure that `bootstrap.sh` is designed to handle termination signals and clean up its child processes appropriately. -#### **Resuming the Import Process** +#### **6.3. Resuming the Import Process** - **Re-run the Bootstrap Script:** ```bash - nohup setsid ./bootstrap.sh 8 /path/to/db_export > /dev/null 2>> bootstrap.log & + setsid ./bootstrap.sh 8 /path/to/db_export > /dev/null 2>> bootstrap.log & ``` - The script will resume where it left off, skipping files that have already been imported successfully. + - Add the `--full` flag if you were using full database mode. -#### **Start the Mirrornode Importer** +#### **6.4. Start the Mirrornode Importer** - Once the bootstrap process completes without errors, you may start the Mirrornode Importer. @@ -301,15 +364,15 @@ If you need to stop the script before it completes: ## Handling Failed Imports -During the import process, the script generates a file named `bootstrap_tracking.txt`, which logs the status of each file import. Each line in this file contains the path and name of a file, followed by its import status: `NOT_STARTED`, `IN_PROGRESS`, `IMPORTED`, or `FAILED_TO_IMPORT`. +During the import process, the script generates a file named `bootstrap_tracking.txt`, which logs the status of each file import. Each line in this file contains the path and name of a file, followed by its import and hash verification status (see [Monitoring and Managing the Import Process](#6-monitoring-and-managing-the-import-process) for status descriptions). **Example of `bootstrap_tracking.txt`:** ``` -/path/to/db_export/record_file.csv.gz IMPORTED -/path/to/db_export/transaction/transaction_part_1.csv.gz IMPORTED -/path/to/db_export/transaction/transaction_part_2.csv.gz FAILED_TO_IMPORT -/path/to/db_export/account.csv.gz NOT_STARTED +/path/to/db_export/record_file.csv.gz IMPORTED HASH_VERIFIED +/path/to/db_export/transaction/transaction_part_1.csv.gz IMPORTED HASH_VERIFIED +/path/to/db_export/transaction/transaction_part_2.csv.gz FAILED_TO_IMPORT HASH_FAILED +/path/to/db_export/account.csv.gz NOT_STARTED HASH_UNVERIFIED ``` **Notes on Data Consistency:** @@ -351,6 +414,7 @@ During the import process, the script generates a file named `bootstrap_tracking - Review `bootstrap.log` for detailed error messages. - Check `bootstrap_tracking.txt` to identify which files failed to import. + - Check `bootstrap_discrepancies.log` for any data verification issues (this file is only created if discrepancies are found in file size, row count, or BLAKE3 hash verification). - Re-run the `bootstrap.sh` script to retry importing failed files. - **Permission Denied Errors:** @@ -365,7 +429,7 @@ During the import process, the script generates a file named `bootstrap_tracking - **Script Does Not Continue After SSH Disconnect:** - - Ensure you used `nohup` when running the script. + - Ensure you used `setsid` when running the script. - Confirm that the script is running by checking the process list: ```bash diff --git a/docs/design/block-streams.md b/docs/design/block-streams.md index 24044c3b163..40d9e3fbc88 100644 --- a/docs/design/block-streams.md +++ b/docs/design/block-streams.md @@ -113,29 +113,6 @@ public class BlockFileTransformer implements StreamFileTransformer blockItems); - - /** - * Block hashes are not included in the block. They must be calculated from the contents of the block. - * - * The `previousBlockHash` is located in the BlockHeader - * The `inputTreeRootHash` is located in the BlockStreamInfo of the last StateChange of the block - * The `startOfBlockStateRootHash` is located in the BlockProof - */ - private byte[] calculateBlockHash( - byte[] previousBlockHash, - byte[] inputTreeRootHash, - byte[] outputTreeRootHash, - byte[] startOfBlockStateRootHash - ); - // The transaction hash will not be included in the block stream output so we will need to calculate it private byte[] calculateTransactionHash(EventTransaction transaction); } @@ -177,32 +154,32 @@ public class BlockStreamPoller extends StreamPoller { } ``` -#### StreamFileNotifier - --Rename `verified` to `notify` as blocks will not be verifiable until each state change has been processed by the BlockStreamVerifier. - #### BlockStreamVerifier ```java package com.hedera.mirror.importer.downloader.block; public class BlockStreamVerifier { - private final StreamFileNotifier streamFileNotifier; private final StreamFileTransformer blockFileTransformer; private final RecordFileParser recordFileParser; + private final StreamFileNotifier streamFileNotifier; /** - * Transforms the block file into a record file, verifies the hash chain and then parses it + * Verifies the block file, transforms it into a record file, and then notifies the parser */ - public void notify(@Nonnull StreamFile streamFile); + public void verify(@NotNull BlockFile blockFile); /** - * For Block N the hash must be verified to match the previousBlockHash protobuf value provided by Block N+1 + * Verifies the block number of the block file + * - that the block number is one after the previous block number if exists + * - that the block number from the file name matches the block number in the block */ - private void verifyHashChain(String expected, String actual); + private void verifyBlockNumber(BlockFile blockFile); - // Verifies that the number of the block file contained in its file name matches the block number within the block file - private void verifyBlockNumber(String expected, String actual); + /** + * The previous hash from the block must match the hash of the previous block + */ + private void verifyHashChain(BlockFile blockFile); } ``` @@ -223,7 +200,7 @@ alter table if exists record_file add column if not exists round_end bigint null; alter table if exists topic_message - alter column if exists running_hash_version drop not null; + alter column running_hash_version drop not null; ``` ### Block to Record File Transformation diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/block/BlockStreamVerifier.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/block/BlockStreamVerifier.java new file mode 100644 index 00000000000..488bce6ce4c --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/downloader/block/BlockStreamVerifier.java @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.importer.downloader.block; + +import com.hedera.mirror.common.domain.transaction.BlockFile; +import com.hedera.mirror.importer.downloader.StreamFileNotifier; +import com.hedera.mirror.importer.exception.HashMismatchException; +import com.hedera.mirror.importer.exception.InvalidStreamFileException; +import com.hedera.mirror.importer.repository.RecordFileRepository; +import jakarta.inject.Named; +import jakarta.validation.constraints.NotNull; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import lombok.RequiredArgsConstructor; +import org.apache.commons.io.FilenameUtils; + +@Named +@RequiredArgsConstructor +public class BlockStreamVerifier { + + private static final BlockFile EMPTY = BlockFile.builder().build(); + + private final BlockFileTransformer blockFileTransformer; + private final RecordFileRepository recordFileRepository; + private final StreamFileNotifier streamFileNotifier; + + private final AtomicReference> lastBlockFile = new AtomicReference<>(Optional.empty()); + + public void verify(@NotNull BlockFile blockFile) { + verifyBlockNumber(blockFile); + verifyHashChain(blockFile); + var recordFile = blockFileTransformer.transform(blockFile); + streamFileNotifier.verified(recordFile); + setLastBlockFile(blockFile); + } + + private Optional getExpectedPreviousHash() { + return getLastBlockFile().map(BlockFile::getHash); + } + + private Optional getLastBlockFile() { + return lastBlockFile.get().or(() -> { + var last = recordFileRepository + .findLatest() + .map(r -> BlockFile.builder() + .hash(r.getHash()) + .index(r.getIndex()) + .build()) + .or(() -> Optional.of(EMPTY)); + lastBlockFile.compareAndSet(Optional.empty(), last); + return last; + }); + } + + private Optional getPreviousBlockNumber() { + return getLastBlockFile().map(BlockFile::getIndex); + } + + private void setLastBlockFile(BlockFile blockFile) { + var copy = (BlockFile) blockFile.copy(); + copy.clear(); + lastBlockFile.set(Optional.of(copy)); + } + + private void verifyBlockNumber(BlockFile blockFile) { + var blockNumber = blockFile.getIndex(); + getPreviousBlockNumber().ifPresent(previousBlockNumber -> { + if (blockNumber != previousBlockNumber + 1) { + throw new InvalidStreamFileException(String.format( + "Non-consecutive block number, previous = %d, current = %d", previousBlockNumber, blockNumber)); + } + }); + + try { + String filename = blockFile.getName(); + int endIndex = filename.indexOf(FilenameUtils.EXTENSION_SEPARATOR); + long actual = Long.parseLong(endIndex != -1 ? filename.substring(0, endIndex) : filename); + if (actual != blockNumber) { + throw new InvalidStreamFileException(String.format( + "Block number mismatch, from filename = %d, from content = %d", actual, blockNumber)); + } + } catch (NumberFormatException e) { + throw new InvalidStreamFileException("Failed to parse block number from filename " + blockFile.getName()); + } + } + + private void verifyHashChain(BlockFile blockFile) { + getExpectedPreviousHash().ifPresent(expected -> { + if (!blockFile.getPreviousHash().contentEquals(expected)) { + throw new HashMismatchException(blockFile.getName(), expected, blockFile.getPreviousHash(), "Previous"); + } + }); + } +} diff --git a/hedera-mirror-importer/src/main/resources/db/scripts/bootstrap.sh b/hedera-mirror-importer/src/main/resources/db/scripts/bootstrap.sh old mode 100644 new mode 100755 index f17d5c69a70..55ea3eab264 --- a/hedera-mirror-importer/src/main/resources/db/scripts/bootstrap.sh +++ b/hedera-mirror-importer/src/main/resources/db/scripts/bootstrap.sh @@ -1,53 +1,69 @@ -#!/bin/bash +#!/usr/bin/env bash -# Write the script's own PID to bootstrap.pid -PID_FILE="bootstrap.pid" -echo $$ > $PID_FILE - -# Enable job control +# Start a new process group and detach from terminal set -m +exec 1>/dev/null 2>>bootstrap.log +[[ -t 1 ]] && exec &0 2>&0 + +# Global variables +CLEANUP_IN_PROGRESS_FILE="bootstrap.cleanup" +export CLEANUP_IN_PROGRESS_FILE + +# Save original stderr and redirect it to suppress job control messages +exec 3>&2 +exec 2>/dev/null + +# Write script's PID to $PID_FILE for process management +PID_FILE="bootstrap.pid" +echo $$ > "$PID_FILE" #################################### # Variables #################################### -# Define minimum required Bash version +# Minimum required Bash version REQUIRED_BASH_MAJOR=4 REQUIRED_BASH_MINOR=3 -# Logging and tracking files -LOG_FILE="bootstrap.log" -TRACKING_FILE="bootstrap_tracking.txt" -LOCK_FILE="bootstrap_tracking.lock" -DISCREPANCY_FILE="discrepancies.log" +# Status tracking and logging files +LOG_FILE="bootstrap.log" # Main log file +TRACKING_FILE="bootstrap_tracking.txt" # Import status tracking +LOCK_FILE="bootstrap_tracking.lock" # File locking for thread safety +DISCREPANCY_FILE="bootstrap_discrepancies.log" # Verification failures -# Required tools -REQUIRED_TOOLS=("psql" "gunzip" "realpath" "flock" "curl") +# Required system tools +# Note: A decompressor (rapidgzip, igzip, or gunzip) is checked separately +REQUIRED_TOOLS=("psql" "realpath" "flock" "curl" "b3sum") -# Flag file to skip database initialization -FLAG_FILE="SKIP_DB_INIT" +# Initialize tracking variables +export DECOMPRESS_TOOL="" # Decompression tool to use (rapidgzip, igzip, or gunzip) +export DECOMPRESS_FLAGS="" # Flags for the decompression tool command +export DECOMPRESSOR_CHECKED=false # Track if decompressor check has been performed +MISSING_TOOLS=() # List of missing required tools -# Assign script arguments -DB_CPU_CORES="$1" -IMPORT_DIR="$2" +# Skip database initialization if this file exists +export DB_SKIP_FLAG_FILE="SKIP_DB_INIT" # Flag file to skip database initialization -# Convert IMPORT_DIR to an absolute path -IMPORT_DIR="$(realpath "$IMPORT_DIR")" +# Manifest selection (manifest.csv or manifest.minimal.csv) +USE_FULL_DB="" # Use full manifest flag +MANIFEST_FILE="" # Path to the manifest file -# Calculate available CPU cores -AVAILABLE_CORES=$(($(nproc) - 1)) # Leave one core free for the local system -DB_AVAILABLE_CORES=$((DB_CPU_CORES - 1)) # Leave one core free for the DB instance +# Parallel processing configuration +B3SUM_NUM_THREADS=1 # Number of threads for BLAKE3 hash calculation +MAX_JOBS="" # Maximum number of concurrent import jobs -# Set file paths -MANIFEST_FILE="${IMPORT_DIR}/manifest.csv" -MIRRORNODE_VERSION_FILE="$IMPORT_DIR/MIRRORNODE_VERSION" +# Associative array for manifest row counts +declare -A manifest_counts # Map of filename to expected row count -declare -A manifest_counts +# Process tracking arrays +declare -a pids=() # List of background process IDs +declare -A processing_files=() # Map of files currently being processed #################################### # Functions #################################### +# Enable/disable pipefail for error handling enable_pipefail() { set -euo pipefail } @@ -58,105 +74,234 @@ disable_pipefail() { export -f enable_pipefail disable_pipefail -# Log messages with UTC timestamps +# Log messages with UTC timestamps to $LOG_FILE log() { local msg="$1" local level="${2:-INFO}" + + # During cleanup, only log messages from cleanup itself + if [[ -f "$CLEANUP_IN_PROGRESS_FILE" && "$level" != "TERMINATE" ]]; then + return + fi + local timestamp timestamp=$(date -u '+%Y-%m-%d %H:%M:%S') echo "[$timestamp] [$level] $msg" >> "$LOG_FILE" } -# Display help message +# Display usage instructions and command-line options directly to the terminal show_help() { - echo "Usage: $0 [OPTIONS] DB_CPU_CORES IMPORT_DIR" - echo - echo "Imports data into a PostgreSQL database from compressed CSV files." - echo - echo "Options:" - echo " -h, --help, -H Show this help message and exit." - echo - echo "Arguments:" - echo " DB_CPU_CORES Number of CPU cores on the DB instance to thread the import jobs." - echo " IMPORT_DIR Path to the directory containing the compressed CSV files." - echo - echo "Example:" - echo " $0 8 /path/to/db_export" - echo + cat > /dev/tty << EOF +Usage: $0 DB_CPU_CORES [--full] IMPORT_DIR + +Imports data into a PostgreSQL database from compressed CSV files. + +Options: + -h, --help, -H Show this help message and exit. + --full Use full database manifest (manifest.csv), otherwise default to minimal (manifest.minimal.csv) + +Arguments: + DB_CPU_CORES Number of CPU cores on the DB instance to thread the import jobs. + IMPORT_DIR Directory containing the compressed CSV files and manifests. + +Example: + # Import minimal database (using manifest.minimal.csv) + $0 8 /path/to/data + + # Import full database (using manifest.csv) + $0 8 --full /path/to/data +EOF } -# Check Bash version +# Verify minimum required Bash version (4.3+) check_bash_version() { local current_major=${BASH_VERSINFO[0]} local current_minor=${BASH_VERSINFO[1]} if (( current_major < REQUIRED_BASH_MAJOR )) || \ (( current_major == REQUIRED_BASH_MAJOR && current_minor < REQUIRED_BASH_MINOR )); then - echo "Error: Bash version ${REQUIRED_BASH_MAJOR}.${REQUIRED_BASH_MINOR}+ is required. Current version is ${BASH_VERSION}." >&2 + log "Bash version ${REQUIRED_BASH_MAJOR}.${REQUIRED_BASH_MINOR}+ is required. Current version is ${BASH_VERSION}." "ERROR" exit 1 fi } -# Kill a process and its descendants +# Verify presence of required system tools and optimal decompressors +check_required_tools() { + # Check required tools + for tool in "${REQUIRED_TOOLS[@]}"; do + if ! command -v "$tool" &> /dev/null; then + MISSING_TOOLS+=("$tool") + fi + done + + # Always check for a decompressor (at minimum gunzip must exist) + if ! $DECOMPRESSOR_CHECKED; then + if ! determine_decompression_tool; then + log "No decompression tool found - at minimum gunzip is required" "ERROR" + log "Recommended tools for faster decompression:" "ERROR" + log " - rapidgzip (fastest): https://github.com/mxmlnkn/rapidgzip" "ERROR" + log " - igzip (next best): https://github.com/intel/isa-l" "ERROR" + fi + DECOMPRESSOR_CHECKED=true + fi + + # Report all missing tools at once if any + if [ "${#MISSING_TOOLS[@]}" -gt 0 ]; then + log "The following required tools are not installed:" "ERROR" + # Use sort -u to remove duplicates + printf "%s\n" "${MISSING_TOOLS[@]}" | sort -u | while read -r tool; do + log " - $tool" "ERROR" + done + log "Please install them to continue." "ERROR" + return 1 + fi + + return 0 +} + +# Select fastest available decompression tool (rapidgzip > igzip > gunzip) +determine_decompression_tool() { + if command -v rapidgzip >/dev/null 2>&1; then + DECOMPRESS_TOOL="rapidgzip" + DECOMPRESS_FLAGS="-d -c -P1" + log "Using rapidgzip for decompression" + return 0 + fi + + if command -v igzip >/dev/null 2>&1; then + DECOMPRESS_TOOL="igzip" + DECOMPRESS_FLAGS="-d -c -T1" + log "Using igzip for decompression" + return 0 + fi + + if command -v gunzip >/dev/null 2>&1; then + DECOMPRESS_TOOL="gunzip" + DECOMPRESS_FLAGS="-c" + log "Using gunzip for decompression" + return 0 + fi + + # No decompression tools found + MISSING_TOOLS+=("gunzip") + return 1 +} + +# Recursively terminate a process and all its child processes kill_descendants() { local pid="$1" local children - children=$(pgrep -P "$pid") + children=$(pgrep -P "$pid" 2>/dev/null) for child in $children; do kill_descendants "$child" done - kill -TERM "$pid" 2>/dev/null + kill -9 "$pid" >/dev/null 2>&1 } -# Handle script termination +# Clean up resources and terminate child processes on script exit cleanup() { disable_pipefail - local trap_type="$1" if [[ "$trap_type" == "INT" || "$trap_type" == "TERM" ]]; then - log "Script interrupted. Terminating background jobs..." "ERROR" + # Redirect stderr to /dev/null for the remainder of cleanup + exec 2>/dev/null + touch "$CLEANUP_IN_PROGRESS_FILE" + + # If a tracking file update was in progress, complete it + if [[ -f "${TRACKING_FILE}.tmp" ]]; then + mv "${TRACKING_FILE}.tmp" "$TRACKING_FILE" + fi + + # First kill all psql processes to stop any active queries + pkill -9 psql + + # Kill all decompression tool processes + pkill -9 "$DECOMPRESS_TOOL" # Kill all background jobs and their descendants for pid in "${pids[@]}"; do - kill_descendants "$pid" + if kill -0 "$pid" 2>/dev/null; then + kill_descendants "$pid" + fi done - wait 2>/dev/null - log "All background jobs terminated." + # Force kill any remaining children immediately + pkill -9 -P $$ + + log "Script interrupted. All processes terminated." "TERMINATE" + + wait # Clean up any zombies + + # Remove files only after ensuring all processes are dead + rm -f "$PID_FILE" "$LOCK_FILE" "$CLEANUP_IN_PROGRESS_FILE" # Exit with a non-zero status to indicate interruption exit 1 fi # Normal cleanup actions (on EXIT) - rm -f "$PID_FILE" "$LOCK_FILE" + rm -f "$PID_FILE" "$LOCK_FILE" "$CLEANUP_IN_PROGRESS_FILE" } -# Safely write to the tracking file with a lock +# Thread-safe update of the tracking file using flock +# Format: filename status hash_status write_tracking_file() { local file="$1" - local status="$2" + local new_status="${2:-}" + local new_hash_status="${3:-}" + ( flock -x 200 - # Remove any existing entry for the file + # Pull existing line (if any) + local existing_line + existing_line=$(grep "^$file " "$TRACKING_FILE" 2>/dev/null || true) + + local old_status="" + local old_hash="" + if [[ -n "$existing_line" ]]; then + old_status=$(echo "$existing_line" | awk '{print $2}') + old_hash=$(echo "$existing_line" | awk '{print $3}') + fi + + # If a line exists but no new import status was passed, keep the old import status + # otherwise default to NOT_STARTED for brand-new file + if [[ -z "$new_status" ]]; then + if [[ -n "$old_status" ]]; then + new_status="$old_status" + else + new_status="NOT_STARTED" + fi + fi + + # If a line exists but no new hash status was passed, keep the old hash status + # otherwise default to HASH_UNVERIFIED for brand-new file + if [[ -z "$new_hash_status" ]]; then + if [[ -n "$old_hash" ]]; then + new_hash_status="$old_hash" + else + new_hash_status="HASH_UNVERIFIED" + fi + fi + + # Remove any existing entry for this file grep -v "^$file " "$TRACKING_FILE" > "${TRACKING_FILE}.tmp" 2>/dev/null || true mv "${TRACKING_FILE}.tmp" "$TRACKING_FILE" - # Add the new status - echo "$file $status" >> "$TRACKING_FILE" + # Add the updated line with the final status and hash + echo "$file $new_status $new_hash_status" >> "$TRACKING_FILE" ) 200>"$LOCK_FILE" } -# Read status from the tracking file +# Read current import status of a file from tracking file read_tracking_status() { local file="$1" - grep "^$file " "$TRACKING_FILE" 2>/dev/null | awk '{print $2}' + grep "^$file " "$TRACKING_FILE" 2>/dev/null | cut -d' ' -f2- } -# Collect all import tasks (compressed CSV files) +# Find all .csv.gz files that need to be imported collect_import_tasks() { enable_pipefail trap 'disable_pipefail' RETURN @@ -164,19 +309,23 @@ collect_import_tasks() { find "$IMPORT_DIR" -type f -name "*.csv.gz" } +# Log file size, row count, or hash verification discrepancies into $DISCREPANCY_FILE write_discrepancy() { local file="$1" local expected_count="$2" local actual_count="$3" - # Only write if not already imported successfully + # Only write if not already imported successfully and not in cleanup discrepancy_entry="$file: expected $expected_count, got $actual_count rows" - if ! grep -q "^${file} IMPORTED$" "$TRACKING_FILE" 2>/dev/null; then + if ! grep -q "^${file} IMPORTED$" "$TRACKING_FILE" 2>/dev/null && [[ ! -f "$CLEANUP_IN_PROGRESS_FILE" ]]; then echo "$discrepancy_entry" >> "$DISCREPANCY_FILE" fi } +# Load PostgreSQL connection settings from $BOOTSTRAP_ENV_FILE (local var) +# Required vars: PGUSER, PGPASSWORD, PGDATABASE, PGHOST, PGPORT source_bootstrap_env() { + # Declare the bootstrap environment file as a local variable local BOOTSTRAP_ENV_FILE="bootstrap.env" if [[ -f "$BOOTSTRAP_ENV_FILE" ]]; then @@ -185,96 +334,98 @@ source_bootstrap_env() { source "$BOOTSTRAP_ENV_FILE" set +a else - log "Error: $BOOTSTRAP_ENV_FILE file not found." "ERROR" + log "$BOOTSTRAP_ENV_FILE file not found." "ERROR" exit 1 fi } +# Parse $MANIFEST_FILE and prepare import tasks +# Format: table_name,file_name,row_count,file_size,blake3_hash process_manifest() { enable_pipefail trap 'disable_pipefail' RETURN # Declare manifest_counts and manifest_tables as global associative arrays + declare -g -A manifest_sizes + declare -g -A manifest_hashes declare -g -A manifest_counts declare -g -A manifest_tables declare -a missing_files=() + log "Using $B3SUM_NUM_THREADS thread(s) per BLAKE3 process." + # Check if the manifest file exists if [[ ! -f "$MANIFEST_FILE" ]]; then - log "Error: Manifest file '$MANIFEST_FILE' not found." "ERROR" + log "Manifest file '$MANIFEST_FILE' not found." "ERROR" exit 1 fi # Validate file count - log "Validating file count" "INFO" + log "Validating file count" # Count files in manifest (excluding header) manifest_file_count=$(tail -n +2 "$MANIFEST_FILE" | wc -l) - # Count actual .gz files in IMPORT_DIR (recursively) + # Count actual files in IMPORT_DIR (recursively) actual_file_count=$(find "$IMPORT_DIR" -type f -name "*.gz" | wc -l) if [[ "$manifest_file_count" != "$actual_file_count" ]]; then log "File count mismatch! Manifest: $manifest_file_count, Directory: $actual_file_count" "ERROR" exit 1 - else - log "File count validation successful" "INFO" fi + log "File count validation successful, Manifest: $manifest_file_count, Directory: $actual_file_count" # Populate manifest_counts, manifest_tables, and run file validations - while IFS=',' read -r filename expected_count expected_size expected_crc32; do + while IFS=',' read -r filename expected_count expected_size expected_blake3_hash; do # Skip header line if [[ "$filename" == "filename" ]]; then continue fi - # Find the file in IMPORT_DIR - file_path=$(find "$IMPORT_DIR" -type f -name "$filename") + # Build file_path based on directory layout + # Large table part files reside in "$IMPORT_DIR//$filename" + if [[ "$filename" =~ ^(.+)_part_ ]]; then + table_prefix="${filename%%_part_*}" + file_path="$IMPORT_DIR/$table_prefix/$filename" + else + # Small table files reside directly in $IMPORT_DIR/ + file_path="$IMPORT_DIR/$filename" + fi if [[ -f "$file_path" ]]; then - # Get CRC32 from GZIP footer and file-size - actual_crc32=$(tail -c 8 "$file_path" | head -c 4 | tr -d '\000' | xxd -p) - actual_size=$(stat -c%s "$file_path") - - # Compare file sizes (strip any whitespace from both values) - if [[ "$actual_size" != "$expected_size" ]]; then - log "File size mismatch for $filename. Expected: $expected_size bytes, Actual: $actual_size bytes" "ERROR" - exit 1 - fi - - # Compare CRC32 values - if [[ "$actual_crc32" != "$expected_crc32" ]]; then - log "CRC32 mismatch for $filename. Expected: $expected_crc32, Actual: $actual_crc32" "ERROR" - exit 1 + # Skip validation if file is already imported successfully + if grep -q "^$file_path IMPORTED" "$TRACKING_FILE" 2>/dev/null; then + continue fi - log "Successfully validated file-size and CRC32 for $filename" - # Skip non-data files and entries with 'N/A' expected count - if [[ "$expected_count" == "N/A" ]]; then - continue - fi - manifest_counts["$filename"]="$expected_count" - - # Extract table name - if [[ "$filename" == "topic_message_low_vol_topic_ids.csv.gz" ]]; then - table="topic_message" - elif [[ "$filename" =~ ^([^/]+)_part_ ]]; then - table="${BASH_REMATCH[1]}" - elif [[ "$filename" =~ ^([^/]+)\.csv\.gz$ ]]; then - table="${BASH_REMATCH[1]}" - else - log "Could not determine table name from filename: $filename" "ERROR" - continue - fi + if [[ "$expected_count" != "N/A" ]]; then + manifest_counts["$filename"]="$expected_count" + + # Extract table name only for data files + if [[ "$filename" =~ ^([^/]+)_part_ ]]; then + table="${BASH_REMATCH[1]}" + elif [[ "$filename" =~ ^([^/]+)\.csv\.gz$ ]]; then + table="${BASH_REMATCH[1]}" + else + log "Could not determine table name from filename: $filename" "ERROR" + continue + fi - # Store table name in manifest_tables - manifest_tables["$table"]=1 + # Store table name in manifest_tables + manifest_tables["$table"]=1 + fi + manifest_sizes["$filename"]="$expected_size" + manifest_hashes["$filename"]="$expected_blake3_hash" else missing_files+=("$filename") fi - done <"$MANIFEST_FILE" + done < "$MANIFEST_FILE" - # If there are missing files, report and exit + MANIFEST_SIZES_SERIALIZED=$(declare -p manifest_sizes) + MANIFEST_HASHES_SERIALIZED=$(declare -p manifest_hashes) + export MANIFEST_SIZES_SERIALIZED MANIFEST_HASHES_SERIALIZED + + # Handle missing files if [[ ${#missing_files[@]} -gt 0 ]]; then log "The following files are listed in the manifest but are missing from the data directory:" "ERROR" for missing_file in "${missing_files[@]}"; do @@ -284,20 +435,236 @@ process_manifest() { fi } -# Initialize the database using init.sh -initialize_database() { +# Validate a file's size and BLAKE3 hash against manifest values +validate_file() { + local file="$1" + local filename="$2" + local failures=() + + if [[ ! -f "$file" ]]; then + echo "file not found" + return 1 + fi + + actual_size=$(stat -c%s "$file") + expected_size="${manifest_sizes["$filename"]}" + + if [[ "$actual_size" != "$expected_size" ]]; then + echo "size mismatch (expected: $expected_size bytes, actual: $actual_size bytes)" + return 1 + fi + + actual_b3sum=$(b3sum --num-threads "$B3SUM_NUM_THREADS" --no-names "$file") + expected_blake3_hash="${manifest_hashes["$filename"]}" + + if [[ "$actual_b3sum" != "$expected_blake3_hash" ]]; then + echo "BLAKE3 hash mismatch (expected: $expected_blake3_hash, actual: $actual_b3sum)" + return 1 + fi + + return 0 +} + +# Validate special files that are required before import can begin +validate_special_files() { enable_pipefail trap 'disable_pipefail' RETURN - # Declare the bootstrap environment file as a local variable - local BOOTSTRAP_ENV_FILE="bootstrap.env" + local special_files=("schema.sql.gz" "MIRRORNODE_VERSION.gz") + local validation_failed=false + local failures=() + + for filename in "${special_files[@]}"; do + local file="$IMPORT_DIR/$filename" + local validation_result + + validation_result=$(validate_file "$file" "$filename") + if [[ $? -ne 0 ]]; then + failures+=("$filename: $validation_result") + validation_failed=true + write_tracking_file "$file" "FAILED_VALIDATION" + else + log "Successfully validated special file: $filename" + write_tracking_file "$file" "IMPORTED" + fi + done + + if [[ "$validation_failed" == "true" ]]; then + log "Special file validation failed:" "ERROR" + for failure in "${failures[@]}"; do + log " - $failure" "ERROR" + done + return 1 + fi + + return 0 +} + +# Import CSV file into database table with post import verification +# Verifies: file size, row count, and BLAKE3 hash +import_file() { + # Don't start new imports during cleanup + if [[ -f "$CLEANUP_IN_PROGRESS_FILE" ]]; then + return 1 + fi + + enable_pipefail + trap 'disable_pipefail' RETURN + + local file="$1" + local table + local filename + local expected_count + local actual_count + local is_small_table + local absolute_file + local start_ts="" + local end_ts="" + local data_suffix="" + local current_pid=$$ + + # Declare manifest_counts, manifest_sizes, and manifest_hashes as associative arrays + declare -A manifest_counts + declare -A manifest_sizes + declare -A manifest_hashes + eval "$MANIFEST_COUNTS_SERIALIZED" + eval "$MANIFEST_SIZES_SERIALIZED" + eval "$MANIFEST_HASHES_SERIALIZED" + + # Get filename and ensure absolute paths + absolute_file="$(realpath "$file")" + filename=$(basename "$absolute_file") + expected_count="${manifest_counts[$filename]}" + + # Mark file as being processed + processing_files["$file"]=1 + + # Perform BLAKE3 and file-size validations + if ! grep -q "^$file IMPORTED" "$TRACKING_FILE" 2>/dev/null; then + local validation_result + validation_result=$(validate_file "$file" "$filename") + if [[ $? -ne 0 ]]; then + if [[ ! -f "$CLEANUP_IN_PROGRESS_FILE" ]]; then + log "Validation failed for $filename: $validation_result" "ERROR" + fi + write_tracking_file "$file" "FAILED_VALIDATION" + return 1 + fi + + log "Successfully validated file-size and BLAKE3 hash for $filename" + current_status=$(read_tracking_status "$file") + if [[ -n "$current_status" ]]; then + status=$(echo "$current_status" | cut -d' ' -f1) + write_tracking_file "$file" "$status" "HASH_VERIFIED" + else + write_tracking_file "$file" "NOT_STARTED" "HASH_VERIFIED" + fi + fi + + # Skip non-table files after validation + if [[ "$filename" == "MIRRORNODE_VERSION.gz" || "$filename" == "schema.sql.gz" ]]; then + log "Successfully validated non-table file: $filename" + write_tracking_file "$file" "IMPORTED" + unset processing_files["$file"] + return 0 + fi + + # Determine if this is a small table by checking filename pattern + is_small_table=false # default to large table part + if [[ ! "$filename" =~ ^(.+)_part_ ]]; then + is_small_table=true # small table + table=$(basename "$file" .csv.gz) + else + # Extract table name - everything before _part_ (can contain underscores) + table="${filename%%_part_*}" + + # Get everything after _part_ and split into components + part_suffix="${filename#*_part_}" # Remove everything up to and including _part_ + start_ts=$(echo "$part_suffix" | cut -d'_' -f2) # Second field after _part_ + end_ts=$(echo "$part_suffix" | cut -d'_' -f3) # Third field after _part_ + data_suffix=$(echo "$part_suffix" | cut -d'_' -f4 | cut -d'.' -f1) # Fourth field, remove extension + fi + + # Log import start and update status + log "Importing into table $table from $filename, PID: $current_pid" + write_tracking_file "$file" "IN_PROGRESS" + + # Execute the import within a transaction + if ! { $DECOMPRESS_TOOL $DECOMPRESS_FLAGS "$file" 2>/dev/null | PGAPPNAME="bootstrap_$current_pid" psql -q -v ON_ERROR_STOP=1 --single-transaction -c "COPY $table FROM STDIN WITH CSV HEADER;" 2>/dev/null; }; then + if [[ ! -f "$CLEANUP_IN_PROGRESS_FILE" ]]; then + log "Error importing data for $file" "ERROR" + fi + write_tracking_file "$file" "FAILED_TO_IMPORT" + return 1 + fi + + # Skip verification if no expected count + if [[ -z "$expected_count" || "$expected_count" == "N/A" ]]; then + log "No expected row count for $filename in manifest, skipping verification." + write_tracking_file "$file" "IMPORTED" + return 0 + fi + + # Row count verification based on table type + if [ "$is_small_table" = true ]; then + if ! actual_count=$(psql -qt -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM \"$table\";" | xargs); then + if [[ ! -f "$CLEANUP_IN_PROGRESS_FILE" ]]; then + log "Error executing count query for $file" "ERROR" + fi + write_tracking_file "$file" "FAILED_TO_IMPORT" + return 1 + fi + else + # Use the previously captured timestamps for large table + if [[ -n "$start_ts" && -n "$end_ts" ]]; then + if ! actual_count=$(psql -qt -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM \"$table\" WHERE consensus_timestamp BETWEEN $start_ts AND $end_ts;" | xargs); then + if [[ ! -f "$CLEANUP_IN_PROGRESS_FILE" ]]; then + log "Error executing count query for $file" "ERROR" + fi + write_tracking_file "$file" "FAILED_TO_IMPORT" + return 1 + fi + else + if [[ ! -f "$CLEANUP_IN_PROGRESS_FILE" ]]; then + log "Error: Missing timestamps for large table file $filename" "ERROR" + fi + write_tracking_file "$file" "FAILED_TO_IMPORT" + return 1 + fi + fi + + # Verify the count matches expected + if [[ "$actual_count" != "$expected_count" ]]; then + if [[ ! -f "$CLEANUP_IN_PROGRESS_FILE" ]]; then + log "Row count mismatch for $file. Expected: $expected_count, Actual: $actual_count" "ERROR" + fi + write_tracking_file "$file" "FAILED_TO_IMPORT" + write_discrepancy "$file" "$expected_count" "$actual_count" + return 1 + else + log "Row count verified, successfully imported $file" + fi + + # Remove file from processing list when done + unset processing_files["$file"] + write_tracking_file "$file" "IMPORTED" + return 0 +} + +# Initialize an empty database schema using init.sh +initialize_database() { + enable_pipefail + trap 'disable_pipefail' RETURN # Reconstruct manifest_tables declare -A manifest_tables eval "$MANIFEST_TABLES_SERIALIZED" - # Source the bootstrap.env file - source_bootstrap_env + # Check for schema.sql + if [[ ! -f "$IMPORT_DIR/schema.sql" ]]; then + log "schema.sql not found in $IMPORT_DIR." "ERROR" + exit 1 + fi # Construct the URL for init.sh INIT_SH_URL="https://raw.githubusercontent.com/hashgraph/hedera-mirror-node/refs/heads/main/hedera-mirror-importer/src/main/resources/db/scripts/init.sh" @@ -308,20 +675,19 @@ initialize_database() { if curl -fSLs -o "init.sh" "$INIT_SH_URL"; then log "Successfully downloaded init.sh" else - log "Error: Failed to download init.sh" "ERROR" + log "Failed to download init.sh" "ERROR" exit 1 fi # Make init.sh executable chmod +x init.sh - # Run init.sh to initialize the database + # Run init.sh to initialize the database and capture its output log "Initializing the database using init.sh" - - if ./init.sh; then + if ./init.sh >> "$LOG_FILE" 2>&1; then log "Database initialized successfully" else - log "Error: Database initialization failed" "ERROR" + log "Database initialization failed. Check $LOG_FILE for details." "ERROR" exit 1 fi @@ -332,38 +698,33 @@ initialize_database() { log "Updated PostgreSQL environment variables to connect to 'mirror_node' database as user 'mirror_node'" - # Set up the schema in the database - if [[ -f "$IMPORT_DIR/schema.sql" ]]; then - log "Executing schema.sql from $IMPORT_DIR" - if psql -v ON_ERROR_STOP=1 -f "$IMPORT_DIR/schema.sql"; then - log "schema.sql executed successfully" - else - log "Error: Failed to execute schema.sql" "ERROR" - exit 1 - fi + # Execute schema.sql + log "Executing schema.sql from $IMPORT_DIR" + if psql -v ON_ERROR_STOP=1 -f "$IMPORT_DIR/schema.sql" >> "$LOG_FILE" 2>&1; then + log "schema.sql executed successfully" else - log "Error: schema.sql not found in $IMPORT_DIR" "ERROR" + log "Failed to execute schema.sql. Check $LOG_FILE for details." "ERROR" exit 1 fi # Check that each table exists in the database # Test database connectivity if ! psql -v ON_ERROR_STOP=1 -c '\q' >/dev/null 2>&1; then - log "Error: Unable to connect to the PostgreSQL database." "ERROR" + log "Unable to connect to the PostgreSQL database." "ERROR" exit 1 fi - log "Successfully connected to the PostgreSQL database." "INFO" + log "Successfully connected to the PostgreSQL database." missing_tables=() declare -A checked_tables_map=() log "Checking table existence in the database" for table in "${!manifest_tables[@]}"; do - log "Verifying existence of table: $table" "INFO" + log "Verifying existence of table: $table" # Avoid duplicate checks if [[ -n "${checked_tables_map["$table"]:-}" ]]; then - log "Table $table has already been checked. Skipping." "INFO" + log "Table $table has already been checked. Skipping." continue fi checked_tables_map["$table"]=1 @@ -373,7 +734,7 @@ initialize_database() { missing_tables+=("$table") log "$table missing from database" "ERROR" else - log "$table exists in the database" "INFO" + log "$table exists in the database" fi done @@ -387,163 +748,35 @@ initialize_database() { log "====================================================" "ERROR" exit 1 else - log "All tables exist in the database." "INFO" + log "All tables exist in the database." fi } -# Import a single file into the database -import_file() { - enable_pipefail - trap 'disable_pipefail' RETURN - - local file="$1" - local table - local filename - local expected_count - local actual_count - - # Declare manifest_counts as an associative array, and reconstruct it in each background job from the serialized data - declare -A manifest_counts - eval "$MANIFEST_COUNTS_SERIALIZED" - - # Determine the table name - if [[ "$(dirname "$file")" == "$IMPORT_DIR" ]]; then - # Small table - filename=$(basename "$file") - - # Skip non-table files - if [[ "$filename" == "MIRRORNODE_VERSION" || "$filename" == "schema.sql" ]]; then - log "Skipping non-table file: $filename" "INFO" - return 0 - fi - - # Handle special case for topic_message_low_vol_topic_ids.csv.gz - if [[ "$filename" == "topic_message_low_vol_topic_ids.csv.gz" ]]; then - table="topic_message" - log "Mapped $filename to table $table" "INFO" - else - table=$(basename "$file" .csv.gz) - fi - - # Assign expected_count from manifest_counts - expected_count="${manifest_counts["$filename"]}" - else - # Large table part - filename="$(basename "$file")" - table=$(basename "$(dirname "$file")") - - # Assign expected_count from manifest_counts - expected_count="${manifest_counts["$filename"]}" - fi - - # Update status to IN_PROGRESS - write_tracking_file "$file" "IN_PROGRESS" - log "Importing table $table from $file" - - # Execute the import within a transaction - if gunzip -c "$file" | PGAPPNAME="$filename" psql -q -v ON_ERROR_STOP=1 --single-transaction -c "COPY $table FROM STDIN WITH CSV HEADER;"; then - # Verification - if [[ -z "$expected_count" || "$expected_count" == "N/A" ]]; then - log "No expected row count for $filename in manifest, skipping verification." - write_tracking_file "$file" "IMPORTED" - else - # Special case for 0.111.0 topic_message_low_vol_topic_ids - if [[ "$MIRRORNODE_VERSION" == "0.111.0" && "$table" == "topic_message" && "$filename" == "topic_message_low_vol_topic_ids.csv.gz" ]]; then - local atma_topic_id="1693742" - if ! actual_count=$(psql -qt -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM \"$table\" WHERE topic_id != $atma_topic_id;" | xargs); then - log "Error executing count query for $file" "ERROR" - write_tracking_file "$file" "FAILED_TO_IMPORT" - return 1 - fi - # Common handling for small tables (both versions) - elif [[ "$(dirname "$file")" == "$IMPORT_DIR" ]]; then - if ! actual_count=$(psql -qt -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM \"$table\";" | xargs); then - log "Error executing count query for $file" "ERROR" - write_tracking_file "$file" "FAILED_TO_IMPORT" - return 1 - fi - # Large table handling (version specific) - else - local basename - basename=$(basename "$file" .csv.gz) - if [[ "$MIRRORNODE_VERSION" == "0.111.0" ]]; then - if [[ "$basename" =~ ^${table}_part_[0-9]+_([0-9]+)_([0-9]+)$ ]]; then - local start_ts="${BASH_REMATCH[1]}" - local end_ts="${BASH_REMATCH[2]}" - - if ! actual_count=$(psql -qt -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM \"$table\" WHERE consensus_timestamp BETWEEN $start_ts AND $end_ts;" | xargs); then - log "Error executing count query for $file" "ERROR" - write_tracking_file "$file" "FAILED_TO_IMPORT" - return 1 - fi - log "Counted rows for table $table within timestamp range $start_ts to $end_ts" "INFO" - else - log "Error parsing timestamps from filename $basename" "ERROR" - write_tracking_file "$file" "FAILED_TO_IMPORT" - fi - # Newer versions handling - else - if [[ "$basename" =~ ^${table}_part_[0-9]+_([0-9]+)_([0-9]+)(_([0-9]+))?$ ]]; then - local start_ts="${BASH_REMATCH[1]}" - local end_ts="${BASH_REMATCH[2]}" - local topic_id="${BASH_REMATCH[4]}" - - if [[ "$table" == "topic_message" && -n "$topic_id" ]]; then - # Topic message with topic_id suffix - if ! actual_count=$(psql -qt -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM \"$table\" WHERE topic_id = $topic_id AND consensus_timestamp BETWEEN $start_ts AND $end_ts;" | xargs); then - log "Error executing count query for $file" "ERROR" - write_tracking_file "$file" "FAILED_TO_IMPORT" - return 1 - fi - else - # Other part files - if ! actual_count=$(psql -qt -v ON_ERROR_STOP=1 -c "SELECT COUNT(*) FROM \"$table\" WHERE consensus_timestamp BETWEEN $start_ts AND $end_ts;" | xargs); then - log "Error executing count query for $file" "ERROR" - write_tracking_file "$file" "FAILED_TO_IMPORT" - return 1 - fi - fi - else - log "Error parsing timestamps from filename $basename" "ERROR" - write_tracking_file "$file" "FAILED_TO_IMPORT" - fi - fi - fi - fi - - # Verify the count matches expected - if [[ "$actual_count" != "$expected_count" ]]; then - log "Row count mismatch for $file. Expected: $expected_count, Actual: $actual_count" "ERROR" - write_tracking_file "$file" "FAILED_TO_IMPORT" - write_discrepancy "$file" "$expected_count" "$actual_count" - return 1 - fi - fi - - write_tracking_file "$file" "IMPORTED" - log "Row count verified, successfully imported $file" "INFO" -} - #################################### # Execution #################################### -# Trap SIGINT and SIGTERM to handle interruptions +# Trap SIGINT and SIGTERM for graceful shutdown trap 'cleanup INT' SIGINT trap 'cleanup TERM' SIGTERM -# Trap EXIT to perform normal cleanup without logging interruption +# Trap EXIT for normal cleanup without interruption logging trap 'cleanup EXIT' EXIT # Perform the Bash version check check_bash_version +# Log the Process Group ID +PGID=$(ps -o pgid= $$ | tr -d ' ') +log "Script Process Group ID: $PGID" + # Display help if no arguments are provided if [[ $# -eq 0 ]]; then echo "No arguments provided. Use --help or -h for usage information." + show_help exit 1 fi -# Parse options +# Parse help first, then options while [[ "$#" -gt 0 ]]; do case $1 in -h | --help | -H) @@ -554,66 +787,106 @@ while [[ "$#" -gt 0 ]]; do break ;; esac + shift done -# Check if required arguments are supplied +# Assign script arguments +DB_CPU_CORES="$1" +shift + +# Process additional options +while [[ $# -gt 0 ]]; do + case $1 in + --full) + USE_FULL_DB=1 + shift + ;; + *) + IMPORT_DIR="$1" + shift + ;; + esac +done + +# Check if required arguments are supplied and valid if [[ -z "$DB_CPU_CORES" || -z "$IMPORT_DIR" ]]; then - echo "Error: Both DB_CPU_CORES and IMPORT_DIR must be provided." - echo "Use --help or -h for usage information." - exit 1 + log "Missing required arguments. Use --help or -h for usage information" "ERROR" + exit 1 fi -# Check if IMPORT_DIR exists and is a directory -if [[ ! -d "$IMPORT_DIR" ]]; then - echo "Error: IMPORT_DIR '$IMPORT_DIR' does not exist or is not a directory." - exit 1 +# Validate DB_CPU_CORES is a positive integer +if ! [[ "$DB_CPU_CORES" =~ ^[1-9][0-9]*$ ]]; then + log "DB_CPU_CORES must be a positive integer" "ERROR" + exit 1 fi -# Check if required tools are installed -missing_tools=() -for tool in "${REQUIRED_TOOLS[@]}"; do - if ! command -v "$tool" &> /dev/null; then - missing_tools+=("$tool") - fi -done +# Convert IMPORT_DIR to an absolute path +IMPORT_DIR="$(realpath "$IMPORT_DIR")" -if [[ ${#missing_tools[@]} -gt 0 ]]; then - echo "Error: The following required tools are not installed:" - for tool in "${missing_tools[@]}"; do - echo " - $tool" - done - echo "Please install them to continue." +# Calculate available CPU cores +AVAILABLE_CORES=$(($(nproc) - 1)) # Leave one core free for the local system +DB_AVAILABLE_CORES=$((DB_CPU_CORES - 1)) # Leave one core free for the DB instance +if (( DB_AVAILABLE_CORES <= 0 )); then + DB_AVAILABLE_CORES=1 # Minimum 1 core +fi +log "DB_AVAILABLE_CORES set to: $DB_AVAILABLE_CORES" + +# Source bootstrap environment variables early +source_bootstrap_env + +# Check for all required tools before proceeding +if ! check_required_tools; then + exit 1 +fi + +# Set file paths +if [[ -n "$USE_FULL_DB" ]]; then + MANIFEST_FILE="${IMPORT_DIR}/manifest.csv" +else + MANIFEST_FILE="${IMPORT_DIR}/manifest.minimal.csv" +fi +MIRRORNODE_VERSION_FILE="$IMPORT_DIR/MIRRORNODE_VERSION" +log "Using manifest file: $MANIFEST_FILE" + +# Check if IMPORT_DIR exists and is a directory +if [[ ! -d "$IMPORT_DIR" ]]; then + log "IMPORT_DIR '$IMPORT_DIR' does not exist or is not a directory" "ERROR" exit 1 fi -# Adjust max_jobs based on system limits +# Calculate optimal number of parallel jobs if [[ $AVAILABLE_CORES -lt $DB_AVAILABLE_CORES ]]; then - max_jobs="$AVAILABLE_CORES" + MAX_JOBS="$AVAILABLE_CORES" else - max_jobs="$DB_AVAILABLE_CORES" + MAX_JOBS="$DB_AVAILABLE_CORES" fi # Process the manifest and check for missing files process_manifest +# Validate special files first +if ! validate_special_files; then + exit 1 +fi + # Decompress schema.sql and MIRRORNODE_VERSION for file in "$IMPORT_DIR/schema.sql.gz" "$IMPORT_DIR/MIRRORNODE_VERSION.gz"; do - if ! gunzip -k -f "$file"; then - log "Error decompressing $file" "ERROR" + if ! $DECOMPRESS_TOOL ${DECOMPRESS_FLAGS/-c/-k} -f "$file" 2>/dev/null; then + log "Error decompressing $file using $DECOMPRESS_TOOL" "ERROR" exit 1 fi done -# Serialize manifest_counts & manifest_tables arrays for export into the subshells +# Serialize manifest data for subshell access MANIFEST_COUNTS_SERIALIZED=$(declare -p manifest_counts) MANIFEST_TABLES_SERIALIZED=$(declare -p manifest_tables) -# Grab the compatible mirrornode version +# Extract and validate mirrornode version if [[ -f "$MIRRORNODE_VERSION_FILE" ]]; then MIRRORNODE_VERSION=$(tr -d '[:space:]' < "$MIRRORNODE_VERSION_FILE") log "Compatible Mirrornode version: $MIRRORNODE_VERSION" else - echo "Error: MIRRORNODE_VERSION file not found in $IMPORT_DIR." + log "MIRRORNODE_VERSION file not found in $IMPORT_DIR." "ERROR" exit 1 fi @@ -621,13 +894,10 @@ fi log "Starting DB import." # Initialize the database unless the flag file exists -if [[ ! -f "$FLAG_FILE" ]]; then +if [[ ! -f "$DB_SKIP_FLAG_FILE" ]]; then initialize_database - touch "$FLAG_FILE" # Create a flag to skip subsequent runs from running db init after it succeeded once + touch "$DB_SKIP_FLAG_FILE" # Create a flag to skip subsequent runs from running db init after it succeeded once else - # Source the bootstrap.env to set OWNER_PASSWORD - source_bootstrap_env - # Set PostgreSQL environment variables export PGUSER="mirror_node" export PGDATABASE="mirror_node" @@ -635,106 +905,97 @@ else log "Set PGUSER, PGDATABASE, and PGPASSWORD for PostgreSQL." - # Validate that the database is already initialized - if psql -U mirror_node -d mirror_node -c "\q" 2>/dev/null; then - log "Skipping database initialization as '$FLAG_FILE' exists." - else - log "Error: Database is not initialized. Cannot skip database initialization." "ERROR" + # Test database connectivity + if ! psql -v ON_ERROR_STOP=1 -c '\q' >/dev/null 2>&1; then + log "Database is not initialized. Cannot skip database initialization." "ERROR" exit 1 fi + log "Database is already initialized, skipping initialization." fi # Get the list of files to import mapfile -t files < <(collect_import_tasks) -# Initialize the tracking file with all files as NOT_STARTED +# Initialize the tracking file with all files as NOT_STARTED and HASH_UNVERIFIED ( flock -x 200 for file in "${files[@]}"; do # Only add if not already in tracking file if ! grep -q "^$file " "$TRACKING_FILE" 2>/dev/null; then - echo "$file NOT_STARTED" >> "$TRACKING_FILE" + echo "$file NOT_STARTED HASH_UNVERIFIED" >> "$TRACKING_FILE" fi done ) 200>"$LOCK_FILE" # Initialize variables for background processes -pids=() -overall_success=1 +overall_success=true +failed_imports=0 -# Export necessary functions and variables for subshells -export -f import_file log kill_descendants write_tracking_file read_tracking_status process_manifest source_bootstrap_env -export IMPORT_DIR LOG_FILE TRACKING_FILE LOCK_FILE MANIFEST_COUNTS_SERIALIZED DISCREPANCY_FILE MIRRORNODE_VERSION +# Export required functions and variables for subshell usage +export -f \ + log show_help check_bash_version check_required_tools \ + determine_decompression_tool kill_descendants cleanup write_tracking_file read_tracking_status \ + collect_import_tasks write_discrepancy source_bootstrap_env process_manifest validate_file \ + validate_special_files initialize_database import_file -# Loop through files and manage parallel execution +export \ + DECOMPRESS_TOOL DECOMPRESS_FLAGS BOOTSTRAP_ENV_FILE DISCREPANCY_FILE \ + IMPORT_DIR LOG_FILE MANIFEST_FILE TRACKING_FILE LOCK_FILE MAX_JOBS + +# Process files in parallel up to $MAX_JOBS for file in "${files[@]}"; do # Check if the file has already been imported - status=$(read_tracking_status "$file") - if [[ "$status" == "IMPORTED" ]]; then - log "Skipping already imported file $file" + if grep -q "^$file IMPORTED" "$TRACKING_FILE" 2>/dev/null; then + log "Skipping processing of already imported file: $file" continue fi - # Wait if max_jobs are already running - while [[ ${#pids[@]} -ge $max_jobs ]]; do + # Wait if $MAX_JOBS are already running + while [[ $(jobs -rp | wc -l) -ge $MAX_JOBS ]]; do # Wait for any job to finish if ! wait -n; then - overall_success=0 + overall_success=false + log "One or more import jobs failed" "ERROR" + ((failed_imports++)) fi - - # Remove completed PIDs from the array - new_pids=() - for pid in "${pids[@]}"; do - if kill -0 "$pid" 2>/dev/null; then - new_pids+=("$pid") - fi - done - pids=("${new_pids[@]}") done - # Start import in background + # Start import in background and capture its PID import_file "$file" & pids+=($!) done # Wait for all remaining jobs to finish -for pid in "${pids[@]}"; do - if ! wait "$pid"; then - overall_success=0 - fi -done +wait # Summarize discrepancies if [[ -s "$DISCREPANCY_FILE" ]]; then - overall_success=0 - echo "====================================================" - echo "Discrepancies detected during import:" - echo "The following files failed the row count verification:" - echo + overall_success=false + log "====================================================" + log "Discrepancies detected during import:" + log "The following files failed the row count verification:" + log "" while read -r line; do - echo "- $line" + log "- $line" done < "$DISCREPANCY_FILE" - echo "====================================================" + log "====================================================" else log "No discrepancies detected during import." - echo "No discrepancies detected during import." fi # Log the final status of the import process -if [[ $overall_success -eq 1 ]]; then - log "DB import completed successfully. The database is fully identical to the data files." - echo "====================================================" - echo "DB import completed successfully." - echo "The database is fully identical to the data files." - echo "====================================================" +if [[ $overall_success = true ]]; then + log "====================================================" + log "DB import completed successfully." + log "The database is fully identical to the data files." + log "====================================================" else - log "The database import process encountered errors and is incomplete. Mirrornode requires a fully synchronized database." "ERROR" - echo "====================================================" - echo "The database import process encountered errors and is incomplete." - echo "Mirrornode requires a fully synchronized database." - echo "Please review the discrepancies above." - echo "====================================================" + log "====================================================" + log "The database import process encountered errors and is incomplete." "ERROR" + log "Mirrornode requires a fully synchronized database." "ERROR" + log "Please review the discrepancies above." "ERROR" + log "====================================================" fi -# Cleanup pid file -rm -f $PID_FILE $LOCK_FILE +# Exit with the appropriate status +exit $((1 - overall_success)) \ No newline at end of file diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/block/BlockStreamVerifierTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/block/BlockStreamVerifierTest.java new file mode 100644 index 00000000000..bc5959c4d26 --- /dev/null +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/downloader/block/BlockStreamVerifierTest.java @@ -0,0 +1,209 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.importer.downloader.block; + +import static com.hedera.mirror.common.domain.DigestAlgorithm.SHA_384; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mock.Strictness.LENIENT; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import com.hedera.mirror.common.domain.StreamFile; +import com.hedera.mirror.common.domain.transaction.BlockFile; +import com.hedera.mirror.common.domain.transaction.RecordFile; +import com.hedera.mirror.common.util.DomainUtils; +import com.hedera.mirror.importer.TestUtils; +import com.hedera.mirror.importer.downloader.StreamFileNotifier; +import com.hedera.mirror.importer.exception.HashMismatchException; +import com.hedera.mirror.importer.exception.InvalidStreamFileException; +import com.hedera.mirror.importer.repository.RecordFileRepository; +import java.time.Instant; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class BlockStreamVerifierTest { + + @Mock(strictness = LENIENT) + private BlockFileTransformer blockFileTransformer; + + @Mock + private RecordFileRepository recordFileRepository; + + @Mock + private StreamFileNotifier streamFileNotifier; + + private RecordFile expectedRecordFile; + private BlockStreamVerifier verifier; + + @BeforeEach + void setup() { + verifier = new BlockStreamVerifier(blockFileTransformer, recordFileRepository, streamFileNotifier); + expectedRecordFile = RecordFile.builder().build(); + when(blockFileTransformer.transform(any())).thenReturn(expectedRecordFile); + } + + @Test + void verifyWithEmptyDb() { + // given + when(recordFileRepository.findLatest()).thenReturn(Optional.empty()); + var blockFile = getBlockFile(null); + + // when + verifier.verify(blockFile); + + // then + verify(blockFileTransformer).transform(blockFile); + verify(recordFileRepository).findLatest(); + verify(streamFileNotifier).verified(expectedRecordFile); + + // given next block file + blockFile = getBlockFile(blockFile); + + // when + verifier.verify(blockFile); + + // then + verify(blockFileTransformer).transform(blockFile); + verify(recordFileRepository).findLatest(); + verify(streamFileNotifier, times(2)).verified(expectedRecordFile); + } + + @Test + void verifyWithPreviousFileInDb() { + // given + var previous = getRecordFile(); + when(recordFileRepository.findLatest()).thenReturn(Optional.of(previous)); + var blockFile = getBlockFile(previous); + + // when + verifier.verify(blockFile); + + // then + verify(blockFileTransformer).transform(blockFile); + verify(recordFileRepository).findLatest(); + verify(streamFileNotifier).verified(expectedRecordFile); + + // given next block file + blockFile = getBlockFile(blockFile); + + // when + verifier.verify(blockFile); + + // then + verify(blockFileTransformer).transform(blockFile); + verify(recordFileRepository).findLatest(); + verify(streamFileNotifier, times(2)).verified(expectedRecordFile); + } + + @Test + void blockNumberMismatch() { + // given + when(recordFileRepository.findLatest()).thenReturn(Optional.empty()); + var blockFile = getBlockFile(null); + blockFile.setIndex(blockFile.getIndex() + 1); + + // when, then + assertThatThrownBy(() -> verifier.verify(blockFile)) + .isInstanceOf(InvalidStreamFileException.class) + .hasMessageContaining("Block number mismatch"); + verifyNoInteractions(blockFileTransformer); + verify(recordFileRepository).findLatest(); + verifyNoInteractions(streamFileNotifier); + } + + @Test + void hashMismatch() { + // given + var previous = getRecordFile(); + when(recordFileRepository.findLatest()).thenReturn(Optional.of(previous)); + var blockFile = getBlockFile(previous); + blockFile.setPreviousHash(sha384Hash()); + + // when, then + assertThatThrownBy(() -> verifier.verify(blockFile)) + .isInstanceOf(HashMismatchException.class) + .hasMessageContaining("Previous hash mismatch"); + verifyNoInteractions(blockFileTransformer); + verify(recordFileRepository).findLatest(); + verifyNoInteractions(streamFileNotifier); + } + + private BlockFile getBlockFile(StreamFile previous) { + long blockNumber = previous != null ? previous.getIndex() + 1 : DomainUtils.convertToNanosMax(Instant.now()); + String previousHash = previous != null ? previous.getHash() : sha384Hash(); + return BlockFile.builder() + .hash(sha384Hash()) + .index(blockNumber) + .name(getBlockFilename(blockNumber)) + .previousHash(previousHash) + .build(); + } + + @Test + void malformedFilename() { + // given + when(recordFileRepository.findLatest()).thenReturn(Optional.empty()); + var blockFile = getBlockFile(null); + blockFile.setName(DomainUtils.bytesToHex(TestUtils.generateRandomByteArray(4))); + + // when, then + assertThatThrownBy(() -> verifier.verify(blockFile)) + .isInstanceOf(InvalidStreamFileException.class) + .hasMessageContaining("Failed to parse block number from filename"); + verifyNoInteractions(blockFileTransformer); + verify(recordFileRepository).findLatest(); + verifyNoInteractions(streamFileNotifier); + } + + @Test + void nonConsecutiveBlockNumber() { + // given + var previous = getRecordFile(); + when(recordFileRepository.findLatest()).thenReturn(Optional.of(previous)); + var blockFile = getBlockFile(null); + + // when, then + assertThatThrownBy(() -> verifier.verify(blockFile)) + .isInstanceOf(InvalidStreamFileException.class) + .hasMessageContaining("Non-consecutive block number"); + verifyNoInteractions(blockFileTransformer); + verify(recordFileRepository).findLatest(); + verifyNoInteractions(streamFileNotifier); + } + + private String getBlockFilename(long blockNumber) { + return StringUtils.leftPad(String.valueOf(blockNumber), 36, "0") + ".blk.gz"; + } + + private RecordFile getRecordFile() { + long index = DomainUtils.convertToNanosMax(Instant.now()); + return RecordFile.builder().hash(sha384Hash()).index(index).build(); + } + + private String sha384Hash() { + return DomainUtils.bytesToHex(TestUtils.generateRandomByteArray(SHA_384.getSize())); + } +} diff --git a/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/EntityIdFromStringConverter.java b/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/EntityIdFromStringConverter.java index 76d531bc46f..6ce64977e8d 100644 --- a/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/EntityIdFromStringConverter.java +++ b/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/EntityIdFromStringConverter.java @@ -27,6 +27,15 @@ public class EntityIdFromStringConverter implements Converter { @Override public EntityId convert(String source) { - return StringUtils.hasText(source) ? EntityId.of(source) : null; + if (!StringUtils.hasText(source)) { + return null; + } + + var parts = source.split("\\."); + if (parts.length == 3) { + return EntityId.of(source); + } + + return EntityId.of(Long.parseLong(source)); } } diff --git a/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/RangeFromStringConverter.java b/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/RangeFromStringConverter.java index 5ce977edc49..e1f8eedac92 100644 --- a/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/RangeFromStringConverter.java +++ b/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/RangeFromStringConverter.java @@ -16,8 +16,8 @@ package com.hedera.mirror.restjava.converter; -import com.google.common.collect.BoundType; import com.google.common.collect.Range; +import io.hypersistence.utils.hibernate.type.range.guava.PostgreSQLGuavaRangeType; import jakarta.inject.Named; import java.util.regex.Pattern; import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; @@ -26,12 +26,8 @@ @Named @ConfigurationPropertiesBinding -@SuppressWarnings("java:S5842") // Upper and lower bounds in regex may be empty and must still match. public class RangeFromStringConverter implements Converter> { - private static final String LOWER_CLOSED = "["; - private static final String UPPER_CLOSED = "]"; - - private static final String RANGE_REGEX = "^([\\[(])?(\\d*)?,(\\d*)?([])])$"; + private static final String RANGE_REGEX = "^([\\[(])?(\\d*)?,\\s*(\\d*)?([])])$"; private static final Pattern RANGE_PATTERN = Pattern.compile(RANGE_REGEX); @Override @@ -40,28 +36,12 @@ public Range convert(String source) { return null; } - var matcher = RANGE_PATTERN.matcher(source); - if (!matcher.matches()) { - throw new IllegalArgumentException("Range string is not valid, '%s'".formatted(source)); - } - - var lowerValueStr = matcher.group(2); - var lowerValue = StringUtils.hasText(lowerValueStr) ? Long.parseLong(lowerValueStr) : null; - - var upperValueStr = matcher.group(3); - var upperValue = StringUtils.hasText(upperValueStr) ? Long.parseLong(upperValueStr) : null; - var upperBoundType = UPPER_CLOSED.equals(matcher.group(4)) ? BoundType.CLOSED : BoundType.OPEN; + var cleanedSource = source.replaceAll("\\s", ""); - Range range; - if (lowerValue != null) { - var lowerBoundType = LOWER_CLOSED.equals(matcher.group(1)) ? BoundType.CLOSED : BoundType.OPEN; - range = upperValue != null - ? Range.range(lowerValue, lowerBoundType, upperValue, upperBoundType) - : Range.downTo(lowerValue, lowerBoundType); - } else { - range = upperValue != null ? Range.upTo(upperValue, upperBoundType) : Range.all(); + if (!RANGE_PATTERN.matcher(cleanedSource).matches()) { + throw new IllegalArgumentException("Range string is not valid, '%s'".formatted(source)); } - return range; + return PostgreSQLGuavaRangeType.ofString(cleanedSource, Long::parseLong, Long.class); } } diff --git a/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/StringToLongConverter.java b/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/StringToLongConverter.java new file mode 100644 index 00000000000..5f28674861c --- /dev/null +++ b/hedera-mirror-rest-java/src/main/java/com/hedera/mirror/restjava/converter/StringToLongConverter.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.restjava.converter; + +import com.hedera.mirror.common.domain.entity.EntityId; +import jakarta.inject.Named; +import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; +import org.springframework.core.convert.converter.Converter; +import org.springframework.util.StringUtils; + +@Named +@ConfigurationPropertiesBinding +public class StringToLongConverter implements Converter { + @Override + public Long convert(String source) { + if (!StringUtils.hasText(source)) { + return null; + } + + var parts = source.split("\\."); + if (parts.length == 3) { + return EntityId.of(source).getId(); + } + return Long.parseLong(source); + } +} diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/converter/StringToLongConverterTest.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/converter/StringToLongConverterTest.java new file mode 100644 index 00000000000..9be8b6b8ee7 --- /dev/null +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/converter/StringToLongConverterTest.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.restjava.converter; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +class StringToLongConverterTest { + + @ParameterizedTest(name = "Convert \"{0}\" to Long") + @CsvSource(delimiter = ',', textBlock = """ + 1, 1 + 0, 0 + 0.0.2, 2 + """) + void testConverter(String source, Long expected) { + var converter = new StringToLongConverter(); + Long actual = converter.convert(source); + assertThat(actual).isEqualTo(expected); + } + + @ParameterizedTest(name = "Convert \"{0}\" to Long") + @NullAndEmptySource + void testInvalidSource(String source) { + var converter = new StringToLongConverter(); + assertThat(converter.convert(source)).isNull(); + } + + @ParameterizedTest(name = "Fail to convert \"{0}\" to Long") + @ValueSource(strings = {"bad", "1.557", "5444.0"}) + void testConverterFailures(String source) { + var converter = new StringToLongConverter(); + assertThrows(NumberFormatException.class, () -> converter.convert(source)); + } +} diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/RestSpecTest.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/RestSpecTest.java index 72381423846..9e6d7a2dcbb 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/RestSpecTest.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/RestSpecTest.java @@ -28,20 +28,27 @@ import com.hedera.mirror.restjava.spec.model.RestSpec; import com.hedera.mirror.restjava.spec.model.RestSpecNormalized; import com.hedera.mirror.restjava.spec.model.SpecTestNormalized; +import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; import java.util.stream.Stream; import javax.sql.DataSource; import lombok.SneakyThrows; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.IOFileFilter; +import org.apache.commons.io.filefilter.TrueFileFilter; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.TestFactory; import org.junit.runner.RunWith; import org.skyscreamer.jsonassert.JSONAssert; import org.skyscreamer.jsonassert.JSONCompareMode; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.web.servlet.DispatcherServletRegistrationBean; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.core.io.Resource; import org.springframework.http.HttpStatusCode; @@ -56,48 +63,38 @@ webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"spring.main.allow-bean-definition-overriding=true"}) public class RestSpecTest extends RestJavaIntegrationTest { - + private static final Pattern INCLUDED_SPEC_DIRS = Pattern.compile( + "^(accounts|accounts/\\{id}/allowances.*|accounts/\\{id}/rewards.*|blocks.*|contracts|network/exchangerate.*|network/fees.*|network/stake.*|topics/\\{id}/messages)$"); + private static final String RESPONSE_HEADER_FILE = "responseHeaders.json"; private static final int JS_REST_API_CONTAINER_PORT = 5551; private static final Path REST_BASE_PATH = Path.of("..", "hedera-mirror-rest", "__tests__", "specs"); - private static final List SELECTED_SPECS = List.of( - REST_BASE_PATH.resolve("accounts/{id}/allowances/crypto/alias-not-found.json"), - REST_BASE_PATH.resolve("accounts/{id}/allowances/crypto/no-params.json"), - REST_BASE_PATH.resolve("accounts/{id}/allowances/crypto/all-params.json"), - REST_BASE_PATH.resolve("accounts/{id}/allowances/crypto/all-params.json"), - REST_BASE_PATH.resolve("accounts/{id}/allowances/tokens/empty.json"), - REST_BASE_PATH.resolve("accounts/{id}/allowances/tokens/no-params.json"), - REST_BASE_PATH.resolve("accounts/{id}/allowances/tokens/specific-spender-id.json"), - REST_BASE_PATH.resolve("accounts/{id}/allowances/tokens/spender-id-range-token-id-upper.json"), - REST_BASE_PATH.resolve("accounts/{id}/rewards/no-params.json"), - REST_BASE_PATH.resolve("accounts/{id}/rewards/no-rewards.json"), - REST_BASE_PATH.resolve("accounts/{id}/rewards/specific-timestamp.json"), - REST_BASE_PATH.resolve("accounts/specific-id.json"), - REST_BASE_PATH.resolve("blocks/no-records.json"), - REST_BASE_PATH.resolve("blocks/timestamp-param.json"), - REST_BASE_PATH.resolve("blocks/all-params-together.json"), - REST_BASE_PATH.resolve("blocks/limit-param.json"), - REST_BASE_PATH.resolve("blocks/no-records.json"), - REST_BASE_PATH.resolve("blocks/{id}/hash-64.json"), - REST_BASE_PATH.resolve("blocks/{id}/hash-96.json"), - REST_BASE_PATH.resolve("network/exchangerate/no-params.json"), - REST_BASE_PATH.resolve("network/exchangerate/timestamp-upper-bound.json"), - // Disable the following test cases since it fails in PR #9973 due to the fact that the PR fixes bug in - // network fees endpoint however the test class runs the hedera-mirror-rest:latest image without the fix - // REST_BASE_PATH.resolve("network/fees/no-params.json"), - // REST_BASE_PATH.resolve("network/fees/order.json"), - REST_BASE_PATH.resolve("network/fees/timestamp-not-found.json"), - REST_BASE_PATH.resolve("network/stake/no-params.json"), - REST_BASE_PATH.resolve("topics/{id}/messages/all-params.json"), - REST_BASE_PATH.resolve("topics/{id}/messages/encoding.json"), - REST_BASE_PATH.resolve("topics/{id}/messages/no-params.json"), - REST_BASE_PATH.resolve("topics/{id}/messages/order.json")); + private static final IOFileFilter SPEC_FILE_FILTER = new IOFileFilter() { + @Override + public boolean accept(File file) { + var directory = file.isDirectory() ? file : file.getParentFile(); + var dirName = directory.getPath().replace(REST_BASE_PATH + "/", ""); + return INCLUDED_SPEC_DIRS.matcher(dirName).matches() && !RESPONSE_HEADER_FILE.equals(file.getName()); + } + + @Override + public boolean accept(File dir, String name) { + return accept(dir); + } + }; + private static final List SELECTED_SPECS = + FileUtils.listFiles(REST_BASE_PATH.toFile(), SPEC_FILE_FILTER, TrueFileFilter.INSTANCE).stream() + .map(File::toPath) + .toList(); private final ResourceDatabasePopulator databaseCleaner; private final DataSource dataSource; private final ObjectMapper objectMapper; private final RestClient restClient; private final SpecDomainBuilder specDomainBuilder; + @Autowired + private DispatcherServletRegistrationBean dispatcherServletRegistration; + RestSpecTest( @Value("classpath:cleanup.sql") Resource cleanupSqlResource, DataSource dataSource, @@ -135,6 +132,12 @@ Stream generateTestsFromSpecs() { continue; } + // Skip tests that require rest application config + if (normalizedSpec.setup().config() != null) { + log.info("Skipping spec file: {} (setup not yet supported)", specFilePath); + continue; + } + var normalizedSpecTests = normalizedSpec.tests(); for (var test : normalizedSpecTests) { var testCases = @@ -175,6 +178,9 @@ private void testSpecUrl(String url, SpecTestNormalized specTest, RestSpecNormal .onStatus(HttpStatusCode::is4xxClientError, (req, res) -> { // Override default handling of 4xx errors, and proceed to evaluate the response. }) + .onStatus(HttpStatusCode::is5xxServerError, (req, res) -> { + // Override default handling of 5xx errors, and proceed to evaluate the response. + }) .toEntity(String.class); assertThat(response.getStatusCode().value()).isEqualTo(specTest.responseStatus()); diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/AbstractEntityBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/AbstractEntityBuilder.java index 985528b19cc..b9802f793af 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/AbstractEntityBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/AbstractEntityBuilder.java @@ -17,7 +17,6 @@ package com.hedera.mirror.restjava.spec.builder; import com.google.common.base.CaseFormat; -import com.hedera.mirror.common.domain.entity.EntityId; import com.hedera.mirror.restjava.spec.model.SpecSetup; import jakarta.annotation.Resource; import jakarta.persistence.EntityManager; @@ -25,8 +24,11 @@ import java.lang.reflect.Method; import java.util.Arrays; import java.util.Base64; +import java.util.Collection; +import java.util.HexFormat; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.function.Supplier; @@ -34,51 +36,57 @@ import java.util.stream.Collectors; import lombok.CustomLog; import org.apache.commons.codec.binary.Base32; -import org.apache.tuweni.bytes.Bytes; +import org.apache.commons.lang3.ArrayUtils; import org.springframework.core.convert.ConversionService; import org.springframework.transaction.support.TransactionOperations; import org.springframework.util.CollectionUtils; @CustomLog abstract class AbstractEntityBuilder implements SpecDomainBuilder { - private static final Base32 BASE32 = new Base32(); - private static final Pattern HEX_STRING_PATTERN = Pattern.compile("^(0x)?[0-9A-Fa-f]+$"); - private static final Map, Map> methodCache = new ConcurrentHashMap<>(); /* * Common handy spec attribute value converter functions to be used by subclasses. */ protected static final Function BASE32_CONVERTER = value -> value == null ? null : BASE32.decode(value.toString()); - - protected static final Function ENTITY_ID_TO_LONG_CONVERTER = value -> value == null - ? 0L - : value instanceof String valueStr ? EntityId.of(valueStr).getId() : (long) value; - + private static final Pattern HEX_STRING_PATTERN = Pattern.compile("^(0x)?[0-9A-Fa-f]+$"); protected static final Function HEX_OR_BASE64_CONVERTER = value -> { if (value instanceof String valueStr) { - return HEX_STRING_PATTERN.matcher(valueStr).matches() - ? Bytes.fromHexString(valueStr.startsWith("0x") ? valueStr : "0x" + valueStr) - .toArray() - : Base64.getDecoder().decode(valueStr); + if (HEX_STRING_PATTERN.matcher(valueStr).matches()) { + var cleanValueStr = valueStr.replace("0x", ""); + + if (cleanValueStr.length() % 2 != 0) { + return HexFormat.of().parseHex(cleanValueStr.substring(0, cleanValueStr.length() - 1)); + } + + return HexFormat.of().parseHex(cleanValueStr); + } + return Base64.getDecoder().decode(valueStr); + } + + if (value instanceof Collection valueCollection) { + return ArrayUtils.toPrimitive(valueCollection.stream() + .map(item -> ((Integer) item).byteValue()) + .toArray(Byte[]::new)); } + return value; }; + private static final Map, Map> methodCache = new ConcurrentHashMap<>(); + // Map a synthetic spec attribute name to another attribute name convertable to a builder method name + protected final Map attributeNameMap; + // Map a builder method by name to a specific attribute value converter function + protected final Map> methodParameterConverters; @Resource - private EntityManager entityManager; + protected ConversionService conversionService; @Resource - private TransactionOperations transactionOperations; + private EntityManager entityManager; @Resource - protected ConversionService conversionService; - - // Map a synthetic spec attribute name to another attribute name convertable to a builder method name - protected final Map attributeNameMap; - // Map a builder method by name to a specific attribute value converter function - protected final Map> methodParameterConverters; + private TransactionOperations transactionOperations; protected AbstractEntityBuilder() { this(Map.of()); @@ -98,9 +106,10 @@ protected AbstractEntityBuilder( * Return the required entity builder instance configured with all initial default values which may be * overridden based on further customization using the spec JSON setup. * + * @param builderContext carries state information about the entity being built * @return entity builder */ - protected abstract B getEntityBuilder(); + protected abstract B getEntityBuilder(SpecBuilderContext builderContext); /** * Perform any post customization processing required and produce a final DB entity to be persisted. @@ -119,14 +128,21 @@ protected AbstractEntityBuilder( */ protected abstract Supplier>> getSpecEntitiesSupplier(SpecSetup specSetup); + protected boolean isHistory(Map entityAttributes) { + return Optional.ofNullable(entityAttributes.get("timestamp_range")) + .map(range -> !range.toString().endsWith(",)")) + .orElse(false); + } + @Override public void customizeAndPersistEntities(SpecSetup specSetup) { var specEntities = getSpecEntitiesSupplier(specSetup).get(); if (!CollectionUtils.isEmpty(specEntities)) { specEntities.forEach(specEntity -> transactionOperations.executeWithoutResult(t -> { - var entityBuilder = getEntityBuilder(); + var entityBuilder = getEntityBuilder(new SpecBuilderContext(isHistory(specEntity))); customizeWithSpec(entityBuilder, specEntity); - entityManager.persist(getFinalEntity(entityBuilder, specEntity)); + var entity = getFinalEntity(entityBuilder, specEntity); + entityManager.persist(entity); })); } } diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/AccountBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/AccountBuilder.java index 917fda4f405..2877e433096 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/AccountBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/AccountBuilder.java @@ -16,7 +16,7 @@ package com.hedera.mirror.restjava.spec.builder; -import com.hedera.mirror.common.domain.entity.Entity; +import com.hedera.mirror.common.domain.entity.AbstractEntity; import com.hedera.mirror.common.domain.entity.EntityType; import com.hedera.mirror.restjava.spec.model.SpecSetup; import jakarta.inject.Named; @@ -33,8 +33,8 @@ protected Supplier>> getSpecEntitiesSupplier(SpecSetup } @Override - protected Entity.EntityBuilder getEntityBuilder() { - return super.getEntityBuilder() + protected AbstractEntity.AbstractEntityBuilder getEntityBuilder(SpecBuilderContext builderContext) { + return super.getEntityBuilder(builderContext) .maxAutomaticTokenAssociations(0) .publicKey("4a5ad514f0957fa170a676210c9bdbddf3bc9519702cf915fa6767a40463b96f") .type(EntityType.ACCOUNT); diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/ContractBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/ContractBuilder.java new file mode 100644 index 00000000000..70f812e2661 --- /dev/null +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/ContractBuilder.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.restjava.spec.builder; + +import com.hedera.mirror.common.domain.contract.Contract; +import com.hedera.mirror.restjava.spec.model.SpecSetup; +import jakarta.inject.Named; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +@Named +public class ContractBuilder extends AbstractEntityBuilder> { + private static final Map ATTRIBUTE_NAME_MAP = Map.of("num", "id"); + + public ContractBuilder() { + super(Map.of(), ATTRIBUTE_NAME_MAP); + } + + @Override + protected Contract.ContractBuilder getEntityBuilder(SpecBuilderContext builderContext) { + return Contract.builder(); + } + + @Override + protected Supplier>> getSpecEntitiesSupplier(SpecSetup specSetup) { + return () -> Optional.ofNullable(specSetup.contracts()).orElse(List.of()).stream() + .filter(contract -> !isHistory(contract)) + .toList(); + } + + @Override + protected Contract getFinalEntity(Contract.ContractBuilder builder, Map entityAttributes) { + return builder.build(); + } +} diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/CryptoAllowanceBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/CryptoAllowanceBuilder.java index 134aec792d2..fc1233eb303 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/CryptoAllowanceBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/CryptoAllowanceBuilder.java @@ -35,7 +35,7 @@ protected Supplier>> getSpecEntitiesSupplier(SpecSetup } @Override - protected CryptoAllowance.CryptoAllowanceBuilder getEntityBuilder() { + protected CryptoAllowance.CryptoAllowanceBuilder getEntityBuilder(SpecBuilderContext builderContext) { return CryptoAllowance.builder() .amount(0L) .amountGranted(0L) diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/EntityBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/EntityBuilder.java index 99442820ecd..99121fee194 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/EntityBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/EntityBuilder.java @@ -17,17 +17,21 @@ package com.hedera.mirror.restjava.spec.builder; import com.google.common.collect.Range; +import com.hedera.mirror.common.domain.entity.AbstractEntity; import com.hedera.mirror.common.domain.entity.Entity; +import com.hedera.mirror.common.domain.entity.EntityHistory; import com.hedera.mirror.common.domain.entity.EntityType; import com.hedera.mirror.restjava.spec.model.SpecSetup; import jakarta.inject.Named; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.function.Supplier; @Named -class EntityBuilder extends AbstractEntityBuilder> { +class EntityBuilder extends AbstractEntityBuilder> { private static final Map> METHOD_PARAMETER_CONVERTERS = Map.of( "alias", BASE32_CONVERTER, @@ -40,13 +44,39 @@ class EntityBuilder extends AbstractEntityBuilder>> getSpecEntitiesSupplier(SpecSetup specSetup) { - return specSetup::entities; + + return () -> { + var entities = specSetup.entities(); + + if (entities == null) { + entities = new ArrayList<>(); + } + + var contracts = specSetup.contracts(); + + if (contracts != null) { + for (var contract : contracts) { + var num = contract.getOrDefault("num", 0L); + var contractEntity = new HashMap(); + contractEntity.put("max_automatic_token_associations", 0); + contractEntity.put("memo", "contract memo"); + contractEntity.put("num", num); + contractEntity.put("type", EntityType.CONTRACT); + contractEntity.put("receiver_sig_required", null); + + contractEntity.putAll(contract); + entities.add(contractEntity); + } + } + + return entities; + }; } @Override - protected Entity.EntityBuilder getEntityBuilder() { - return Entity.builder() - .declineReward(Boolean.FALSE) + protected AbstractEntity.AbstractEntityBuilder getEntityBuilder(SpecBuilderContext builderContext) { + var builder = builderContext.isHistory() ? EntityHistory.builder() : Entity.builder(); + return builder.declineReward(Boolean.FALSE) .deleted(Boolean.FALSE) .num(0L) .memo("entity memo") @@ -61,11 +91,17 @@ protected Supplier>> getSpecEntitiesSupplier(SpecSetup } @Override - protected Entity getFinalEntity(Entity.EntityBuilder builder, Map account) { + protected AbstractEntity getFinalEntity( + AbstractEntity.AbstractEntityBuilder builder, Map account) { var entity = builder.build(); if (entity.getId() == null) { - builder.id(entity.toEntityId().getId()); - entity = builder.build(); + entity.setId(entity.toEntityId().getId()); + + // Work around entity builder resetting the public key when setting key + var publicKeyOverride = account.get("public_key"); + if (entity.getPublicKey() == null && publicKeyOverride != null) { + entity.setPublicKey(publicKeyOverride.toString()); + } } return entity; } diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/EntityStakeBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/EntityStakeBuilder.java new file mode 100644 index 00000000000..80e65835569 --- /dev/null +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/EntityStakeBuilder.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.restjava.spec.builder; + +import com.google.common.collect.Range; +import com.hedera.mirror.common.domain.entity.AbstractEntityStake; +import com.hedera.mirror.common.domain.entity.EntityStake; +import com.hedera.mirror.common.domain.entity.EntityStakeHistory; +import com.hedera.mirror.common.util.DomainUtils; +import com.hedera.mirror.restjava.spec.model.SpecSetup; +import jakarta.inject.Named; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +@Named +public class EntityStakeBuilder + extends AbstractEntityBuilder> { + private static final long SECONDS_PER_DAY = 86400; + + @Override + protected AbstractEntityStake.AbstractEntityStakeBuilder getEntityBuilder(SpecBuilderContext builderContext) { + var builder = builderContext.isHistory() ? EntityStakeHistory.builder() : EntityStake.builder(); + return builder.endStakePeriod(1) + .pendingReward(0) + .stakedNodeIdStart(1) + .stakedToMe(0) + .stakeTotalStart(0); + } + + @Override + protected AbstractEntityStake getFinalEntity( + AbstractEntityStake.AbstractEntityStakeBuilder builder, Map entityAttributes) { + + var entityStake = builder.build(); + if (entityStake.getTimestampRange() == null) { + var seconds = SECONDS_PER_DAY * (entityStake.getEndStakePeriod() + 1); + var lowerBound = seconds * DomainUtils.NANOS_PER_SECOND + 1; + entityStake.setTimestampRange(Range.atLeast(lowerBound)); + } + return entityStake; + } + + @Override + protected Supplier>> getSpecEntitiesSupplier(SpecSetup specSetup) { + return specSetup::entityStakes; + } +} diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/FileDataBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/FileDataBuilder.java index 6b23138823e..f1fc76ede75 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/FileDataBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/FileDataBuilder.java @@ -26,7 +26,6 @@ @Named class FileDataBuilder extends AbstractEntityBuilder { - private static final Map> METHOD_PARAMETER_CONVERTERS = Map.of("fileData", HEX_OR_BASE64_CONVERTER); @@ -40,7 +39,7 @@ protected Supplier>> getSpecEntitiesSupplier(SpecSetup } @Override - protected FileData.FileDataBuilder getEntityBuilder() { + protected FileData.FileDataBuilder getEntityBuilder(SpecBuilderContext builderContext) { return FileData.builder().transactionType(17); } diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/NetworkStakeBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/NetworkStakeBuilder.java index 830c1f37e89..a16e71b0fa6 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/NetworkStakeBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/NetworkStakeBuilder.java @@ -32,7 +32,7 @@ protected Supplier>> getSpecEntitiesSupplier(SpecSetup } @Override - protected NetworkStake.NetworkStakeBuilder getEntityBuilder() { + protected NetworkStake.NetworkStakeBuilder getEntityBuilder(SpecBuilderContext builderContext) { return NetworkStake.builder() .consensusTimestamp(0L) .epochDay(0L) diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/RecordFileBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/RecordFileBuilder.java index 84ae5abf586..a571095b769 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/RecordFileBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/RecordFileBuilder.java @@ -24,17 +24,20 @@ import jakarta.inject.Named; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.function.Supplier; @Named class RecordFileBuilder extends AbstractEntityBuilder { - private static final Map ATTRIBUTE_NAME_MAP = Map.of("prev_hash", "previous_hash"); private static final byte[] DEFAULT_BYTES = new byte[] {1, 1, 2, 2, 3, 3}; + private static final Map> METHOD_PARAMETER_CONVERTERS = + Map.of("logsBloom", HEX_OR_BASE64_CONVERTER); + RecordFileBuilder() { - super(Map.of(), ATTRIBUTE_NAME_MAP); + super(METHOD_PARAMETER_CONVERTERS, ATTRIBUTE_NAME_MAP); } @Override @@ -43,7 +46,7 @@ protected Supplier>> getSpecEntitiesSupplier(SpecSetup } @Override - protected RecordFile.RecordFileBuilder getEntityBuilder() { + protected RecordFile.RecordFileBuilder getEntityBuilder(SpecBuilderContext builderContext) { return RecordFile.builder() .bytes(DEFAULT_BYTES) .consensusEnd(1628751573995691000L) @@ -73,7 +76,7 @@ protected RecordFile.RecordFileBuilder getEntityBuilder() { @Override protected RecordFile getFinalEntity(RecordFile.RecordFileBuilder builder, Map account) { var entity = builder.build(); - builder.size(entity.getBytes() != null ? entity.getBytes().length : entity.getSize()); + builder.size(entity.getBytes() != null ? Integer.valueOf(entity.getBytes().length) : entity.getSize()); return builder.build(); } } diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/SpecBuilderContext.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/SpecBuilderContext.java new file mode 100644 index 00000000000..0df0ad4c103 --- /dev/null +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/SpecBuilderContext.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.restjava.spec.builder; + +public record SpecBuilderContext(boolean isHistory) {} diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/StakingRewardTransferBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/StakingRewardTransferBuilder.java index 2583a7665b1..af23b731ba1 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/StakingRewardTransferBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/StakingRewardTransferBuilder.java @@ -22,27 +22,19 @@ import jakarta.inject.Named; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.function.Supplier; @Named class StakingRewardTransferBuilder extends AbstractEntityBuilder { - private static final Map> METHOD_PARAMETER_CONVERTERS = - Map.of("accountId", ENTITY_ID_TO_LONG_CONVERTER); - - StakingRewardTransferBuilder() { - super(METHOD_PARAMETER_CONVERTERS); - } - @Override protected Supplier>> getSpecEntitiesSupplier(SpecSetup specSetup) { return specSetup::stakingRewardTransfers; } @Override - protected StakingRewardTransfer.StakingRewardTransferBuilder getEntityBuilder() { + protected StakingRewardTransfer.StakingRewardTransferBuilder getEntityBuilder(SpecBuilderContext builderContext) { return StakingRewardTransfer.builder().accountId(1001L).amount(100L).payerAccountId(EntityId.of(950L)); } diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TokenAccountBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TokenAccountBuilder.java index 306b57bce3a..aaf2623159d 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TokenAccountBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TokenAccountBuilder.java @@ -22,27 +22,18 @@ import jakarta.inject.Named; import java.util.List; import java.util.Map; -import java.util.function.Function; import java.util.function.Supplier; @Named class TokenAccountBuilder extends AbstractEntityBuilder> { - private static final Map> METHOD_PARAMETER_CONVERTERS = Map.of( - "accountId", ENTITY_ID_TO_LONG_CONVERTER, - "tokenId", ENTITY_ID_TO_LONG_CONVERTER); - - TokenAccountBuilder() { - super(METHOD_PARAMETER_CONVERTERS); - } - @Override protected Supplier>> getSpecEntitiesSupplier(SpecSetup specSetup) { return specSetup::tokenAccounts; } @Override - protected TokenAccount.TokenAccountBuilder getEntityBuilder() { + protected TokenAccount.TokenAccountBuilder getEntityBuilder(SpecBuilderContext builderContext) { return TokenAccount.builder() .accountId(0L) .associated(Boolean.TRUE) diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TokenAllowanceBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TokenAllowanceBuilder.java index 2db24d5bc5c..afb6e0d2260 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TokenAllowanceBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TokenAllowanceBuilder.java @@ -34,7 +34,7 @@ protected Supplier>> getSpecEntitiesSupplier(SpecSetup } @Override - protected TokenAllowance.TokenAllowanceBuilder getEntityBuilder() { + protected TokenAllowance.TokenAllowanceBuilder getEntityBuilder(SpecBuilderContext builderContext) { return TokenAllowance.builder() .amount(0L) .amountGranted(0L) diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TopicMessageBuilder.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TopicMessageBuilder.java index d04b31a1d54..d42938d6612 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TopicMessageBuilder.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/builder/TopicMessageBuilder.java @@ -34,7 +34,7 @@ protected Supplier>> getSpecEntitiesSupplier(SpecSetup } @Override - protected TopicMessage.TopicMessageBuilder getEntityBuilder() { + protected TopicMessage.TopicMessageBuilder getEntityBuilder(SpecBuilderContext builderContext) { return TopicMessage.builder() .message("message".getBytes(StandardCharsets.UTF_8)) .payerAccountId(EntityId.of(3L)) diff --git a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/model/SpecSetup.java b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/model/SpecSetup.java index 27f5c1e0fce..417543f95e4 100644 --- a/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/model/SpecSetup.java +++ b/hedera-mirror-rest-java/src/test/java/com/hedera/mirror/restjava/spec/model/SpecSetup.java @@ -21,12 +21,14 @@ import java.util.Map; public record SpecSetup( - Map features, + Map config, List> accounts, List> contracts, List> cryptoAllowances, @JsonProperty("cryptotransfers") List> cryptoTransfers, List> entities, + List> entityStakes, + Map features, @JsonProperty("filedata") List> fileData, @JsonProperty("networkstakes") List> networkStakes, List> nfts, diff --git a/hedera-mirror-rest/__tests__/specs/network/exchangerate/fallback.json b/hedera-mirror-rest/__tests__/specs/network/exchangerate/fallback.json index 89781ae057f..199b577db45 100644 --- a/hedera-mirror-rest/__tests__/specs/network/exchangerate/fallback.json +++ b/hedera-mirror-rest/__tests__/specs/network/exchangerate/fallback.json @@ -23,7 +23,7 @@ { "consensus_timestamp": 1234567890900800700, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 16 } ] diff --git a/hedera-mirror-rest/__tests__/specs/network/exchangerate/fallbackMaxRetries.json b/hedera-mirror-rest/__tests__/specs/network/exchangerate/fallbackMaxRetries.json index 62bbf578704..9db7b1b40c3 100644 --- a/hedera-mirror-rest/__tests__/specs/network/exchangerate/fallbackMaxRetries.json +++ b/hedera-mirror-rest/__tests__/specs/network/exchangerate/fallbackMaxRetries.json @@ -17,67 +17,67 @@ { "consensus_timestamp": 1234567890900800700, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800701, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800702, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800703, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800704, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800705, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800706, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800707, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800708, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800709, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 }, { "consensus_timestamp": 1234567890900800710, "entity_id": 112, - "file_data": "0a1008corrupt", + "file_data": "0a1008c", "transaction_type": 19 } ] diff --git a/hedera-mirror-web3/src/main/java/com/hedera/hapi/node/state/token/Token.java b/hedera-mirror-web3/src/main/java/com/hedera/hapi/node/state/token/Token.java index 9099abb3f7a..63dd0498621 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/hapi/node/state/token/Token.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/hapi/node/state/token/Token.java @@ -185,7 +185,7 @@ public Token( String name, String symbol, int decimals, - Long totalSupply, + long totalSupply, AccountID treasuryAccountId, Key adminKey, Key kycKey, diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/common/ContractCallContext.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/common/ContractCallContext.java index 04fde3970ec..89a78588b37 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/common/ContractCallContext.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/common/ContractCallContext.java @@ -26,7 +26,9 @@ import com.hedera.mirror.web3.evm.contracts.execution.traceability.OpcodeTracerOptions; import com.hedera.mirror.web3.evm.store.CachingStateFrame; import com.hedera.mirror.web3.evm.store.StackedStateFrames; +import com.hedera.mirror.web3.service.model.CallServiceParameters; import com.hedera.mirror.web3.state.FileReadableKVState; +import com.hedera.mirror.web3.viewmodel.BlockType; import java.util.ArrayList; import java.util.EmptyStackException; import java.util.List; @@ -57,6 +59,10 @@ public class ContractCallContext { @Setter private List opcodes = new ArrayList<>(); + + @Setter + private CallServiceParameters callServiceParameters; + /** * Record file which stores the block timestamp and other historical block details used for filtering of historical * data. @@ -101,7 +107,6 @@ public static T run(Function function) { } public void reset() { - recordFile = null; stack = stackBase; file = Optional.empty(); } @@ -147,7 +152,10 @@ public void initializeStackFrames(final StackedStateFrames stackedStateFrames) { } public boolean useHistorical() { - return recordFile != null; + if (callServiceParameters != null) { + return callServiceParameters.getBlock() != BlockType.LATEST; + } + return recordFile != null; // Remove recordFile comparison after mono code deletion } public void incrementContractActionsCounter() { diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/evm/pricing/RatesAndFeesLoader.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/evm/pricing/RatesAndFeesLoader.java index 12b0877c76e..d80d1f3f86b 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/evm/pricing/RatesAndFeesLoader.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/evm/pricing/RatesAndFeesLoader.java @@ -62,14 +62,7 @@ public class RatesAndFeesLoader { public static final EntityId EXCHANGE_RATE_ENTITY_ID = EntityId.of(0L, 0L, 112L); public static final EntityId FEE_SCHEDULE_ENTITY_ID = EntityId.of(0L, 0L, 111L); - - static final ExchangeRateSet DEFAULT_EXCHANGE_RATE_SET = ExchangeRateSet.newBuilder() - .setCurrentRate(ExchangeRate.newBuilder() - .setCentEquiv(12) - .setExpirationTime(TimestampSeconds.newBuilder().setSeconds(4102444800L)) - .setHbarEquiv(1)) - .build(); - static final CurrentAndNextFeeSchedule DEFAULT_FEE_SCHEDULE = CurrentAndNextFeeSchedule.newBuilder() + public static final CurrentAndNextFeeSchedule DEFAULT_FEE_SCHEDULE = CurrentAndNextFeeSchedule.newBuilder() .setCurrentFeeSchedule(FeeSchedule.newBuilder() .setExpiryTime(TimestampSeconds.newBuilder().setSeconds(4102444800L)) .addTransactionFeeSchedule(TransactionFeeSchedule.newBuilder() @@ -264,6 +257,12 @@ public class RatesAndFeesLoader { .setGas(852000) .build())))) .build(); + static final ExchangeRateSet DEFAULT_EXCHANGE_RATE_SET = ExchangeRateSet.newBuilder() + .setCurrentRate(ExchangeRate.newBuilder() + .setCentEquiv(12) + .setExpirationTime(TimestampSeconds.newBuilder().setSeconds(4102444800L)) + .setHbarEquiv(1)) + .build(); private static final CurrentAndNextFeeSchedule EMPTY_FEE_SCHEDULE = CurrentAndNextFeeSchedule.getDefaultInstance(); private static final ExchangeRateSet EMPTY_EXCHANGE_RATE_SET = ExchangeRateSet.getDefaultInstance(); private final RetryTemplate retryTemplate = RetryTemplate.builder() diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/evm/properties/MirrorNodeEvmProperties.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/evm/properties/MirrorNodeEvmProperties.java index e4155199d83..f00789f0320 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/evm/properties/MirrorNodeEvmProperties.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/evm/properties/MirrorNodeEvmProperties.java @@ -25,10 +25,13 @@ import static com.swirlds.common.utility.CommonUtils.unhex; import static com.swirlds.state.lifecycle.HapiUtils.SEMANTIC_VERSION_COMPARATOR; +import com.google.common.collect.ImmutableSortedMap; import com.hedera.hapi.node.base.SemanticVersion; import com.hedera.mirror.common.domain.entity.EntityType; import com.hedera.mirror.web3.common.ContractCallContext; +import com.hedera.node.app.config.ConfigProviderImpl; import com.hedera.node.app.service.evm.contracts.execution.EvmProperties; +import com.hedera.node.config.VersionedConfiguration; import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotBlank; @@ -64,6 +67,9 @@ @ConfigurationProperties(prefix = "hedera.mirror.web3.evm") public class MirrorNodeEvmProperties implements EvmProperties { + private static final NavigableMap DEFAULT_EVM_VERSION_MAP = + ImmutableSortedMap.of(0L, EVM_VERSION); + @Getter private boolean allowTreasuryToOwnNfts = true; @@ -176,11 +182,17 @@ public class MirrorNodeEvmProperties implements EvmProperties { "contracts.chainId", chainIdBytes32().toBigInteger().toString(), "contracts.maxRefundPercentOfGasLimit", - String.valueOf(maxGasRefundPercentage())); + String.valueOf(maxGasRefundPercentage()), + "contracts.sidecars", + ""); @Getter(lazy = true) private final Map transactionProperties = buildTransactionProperties(); + @Getter(lazy = true) + private final VersionedConfiguration versionedConfiguration = + new ConfigProviderImpl(false, null, getTransactionProperties()).getConfiguration(); + @Getter @Min(1) private int feesTokenTransferUsageMultiplier = 380; @@ -295,7 +307,7 @@ public NavigableMap getEvmVersions() { return network.evmVersions; } - return new TreeMap<>(Map.of(0L, EVM_VERSION)); + return DEFAULT_EVM_VERSION_MAP; } /** @@ -323,15 +335,11 @@ public int feesTokenTransferUsageMultiplier() { } private Map buildTransactionProperties() { - final Map mirrorNodeProperties = new HashMap<>(properties); - mirrorNodeProperties.put( - "contracts.evm.version", - "v" - + getSemanticEvmVersion().major() + "." - + getSemanticEvmVersion().minor()); + var mirrorNodeProperties = new HashMap<>(properties); + mirrorNodeProperties.put("contracts.evm.version", "v" + evmVersion.major() + "." + evmVersion.minor()); mirrorNodeProperties.put( "ledger.id", Bytes.wrap(getNetwork().getLedgerId()).toHexString()); - return mirrorNodeProperties; + return Collections.unmodifiableMap(mirrorNodeProperties); } @Getter diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/exception/MirrorEvmTransactionException.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/exception/MirrorEvmTransactionException.java index 2e9bb3441ba..424ed085229 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/exception/MirrorEvmTransactionException.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/exception/MirrorEvmTransactionException.java @@ -19,13 +19,16 @@ import static org.apache.commons.lang3.StringUtils.EMPTY; import com.hedera.mirror.web3.evm.exception.EvmException; +import com.hedera.node.app.service.evm.contracts.execution.HederaEvmTransactionProcessingResult; import com.hederahashgraph.api.proto.java.ResponseCodeEnum; import java.io.Serial; import java.nio.charset.StandardCharsets; +import lombok.Getter; import org.apache.commons.codec.binary.Hex; import org.apache.commons.lang3.StringUtils; import org.apache.tuweni.bytes.Bytes; +@Getter @SuppressWarnings("java:S110") public class MirrorEvmTransactionException extends EvmException { @@ -34,18 +37,26 @@ public class MirrorEvmTransactionException extends EvmException { private final String detail; private final String data; + private final transient HederaEvmTransactionProcessingResult result; public MirrorEvmTransactionException( final ResponseCodeEnum responseCode, final String detail, final String hexData) { - super(responseCode.name()); - this.detail = detail; - this.data = hexData; + this(responseCode.name(), detail, hexData, null); } public MirrorEvmTransactionException(final String message, final String detail, final String hexData) { + this(message, detail, hexData, null); + } + + public MirrorEvmTransactionException( + final String message, + final String detail, + final String hexData, + HederaEvmTransactionProcessingResult result) { super(message); this.detail = detail; this.data = hexData; + this.result = result; } public Bytes messageBytes() { @@ -53,14 +64,6 @@ public Bytes messageBytes() { return Bytes.of(message.getBytes(StandardCharsets.UTF_8)); } - public String getDetail() { - return detail; - } - - public String getData() { - return data; - } - @Override public String toString() { return "%s(message=%s, detail=%s, data=%s, dataDecoded=%s)" diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/repository/RecordFileRepository.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/repository/RecordFileRepository.java index 9e7decb3273..a3e50ec9a8a 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/repository/RecordFileRepository.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/repository/RecordFileRepository.java @@ -25,6 +25,7 @@ import static com.hedera.mirror.web3.evm.config.EvmConfiguration.CACHE_NAME_RECORD_FILE_LATEST_INDEX; import com.hedera.mirror.common.domain.transaction.RecordFile; +import java.util.List; import java.util.Optional; import org.springframework.cache.annotation.CachePut; import org.springframework.cache.annotation.Cacheable; @@ -65,4 +66,7 @@ public interface RecordFileRepository extends PagingAndSortingRepository= ?1 order by r.consensusEnd asc limit 1") Optional findByTimestamp(long timestamp); + + @Query(value = "select * from record_file where index >= ?1 and index <= ?2 order by index asc", nativeQuery = true) + List findByIndexRange(long startIndex, long endIndex); } diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/ContractCallService.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/ContractCallService.java index 16ad0c85751..9f7451190ae 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/ContractCallService.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/ContractCallService.java @@ -22,6 +22,7 @@ import static com.hedera.mirror.web3.service.model.CallServiceParameters.CallType.ERROR; import static org.apache.logging.log4j.util.Strings.EMPTY; +import com.google.common.annotations.VisibleForTesting; import com.hedera.mirror.web3.common.ContractCallContext; import com.hedera.mirror.web3.evm.contracts.execution.MirrorEvmTxProcessor; import com.hedera.mirror.web3.evm.properties.MirrorNodeEvmProperties; @@ -46,13 +47,14 @@ public abstract class ContractCallService { static final String GAS_LIMIT_METRIC = "hedera.mirror.web3.call.gas.limit"; static final String GAS_USED_METRIC = "hedera.mirror.web3.call.gas.used"; protected final Store store; + protected final MirrorNodeEvmProperties mirrorNodeEvmProperties; private final MeterProvider gasLimitCounter; private final MeterProvider gasUsedCounter; private final MirrorEvmTxProcessor mirrorEvmTxProcessor; private final RecordFileService recordFileService; private final ThrottleProperties throttleProperties; private final Bucket gasLimitBucket; - private final MirrorNodeEvmProperties mirrorNodeEvmProperties; + private final TransactionExecutionService transactionExecutionService; @SuppressWarnings("java:S107") @@ -80,6 +82,12 @@ protected ContractCallService( this.transactionExecutionService = transactionExecutionService; } + @VisibleForTesting + public HederaEvmTransactionProcessingResult callContract(CallServiceParameters params) + throws MirrorEvmTransactionException { + return ContractCallContext.run(context -> callContract(params, context)); + } + /** * This method is responsible for calling a smart contract function. The method is divided into two main parts: *

@@ -99,15 +107,22 @@ protected ContractCallService( */ protected HederaEvmTransactionProcessingResult callContract(CallServiceParameters params, ContractCallContext ctx) throws MirrorEvmTransactionException { - // if we have historical call, then set the corresponding record file in the context - if (params.getBlock() != BlockType.LATEST) { + ctx.setCallServiceParameters(params); + + if (mirrorNodeEvmProperties.isModularizedServices() || params.getBlock() != BlockType.LATEST) { ctx.setRecordFile(recordFileService .findByBlockType(params.getBlock()) .orElseThrow(BlockNumberNotFoundException::new)); } + // initializes the stack frame with the current state or historical state (if the call is historical) - ctx.initializeStackFrames(store.getStackedStateFrames()); - return doProcessCall(params, params.getGas(), true); + if (!mirrorNodeEvmProperties.isModularizedServices()) { + ctx.initializeStackFrames(store.getStackedStateFrames()); + } + + var result = doProcessCall(params, params.getGas(), true); + validateResult(result, params.getCallType()); + return result; } protected HederaEvmTransactionProcessingResult doProcessCall( @@ -152,7 +167,8 @@ protected void validateResult(final HederaEvmTransactionProcessingResult txnResu updateGasUsedMetric(ERROR, txnResult.getGasUsed(), 1); var revertReason = txnResult.getRevertReason().orElse(Bytes.EMPTY); var detail = maybeDecodeSolidityErrorStringToReadableMessage(revertReason); - throw new MirrorEvmTransactionException(getStatusOrDefault(txnResult), detail, revertReason.toHexString()); + throw new MirrorEvmTransactionException( + getStatusOrDefault(txnResult).name(), detail, revertReason.toHexString(), txnResult); } else { updateGasUsedMetric(type, txnResult.getGasUsed(), 1); } diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/ContractExecutionService.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/ContractExecutionService.java index 85547d3cfc6..bb56fbcb0b4 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/ContractExecutionService.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/ContractExecutionService.java @@ -72,14 +72,9 @@ public String processCall(final ContractExecutionParameters params) { Bytes result; if (params.isEstimate()) { - // eth_estimateGas initialization - historical timestamp is Optional.empty() - ctx.initializeStackFrames(store.getStackedStateFrames()); - result = estimateGas(params); + result = estimateGas(params, ctx); } else { final var ethCallTxnResult = callContract(params, ctx); - - validateResult(ethCallTxnResult, params.getCallType()); - result = Objects.requireNonNullElse(ethCallTxnResult.getOutput(), Bytes.EMPTY); } @@ -103,8 +98,8 @@ public String processCall(final ContractExecutionParameters params) { * 2. Finally, if the first step is successful, a binary search is initiated. The lower bound of the search is the * gas used in the first step, while the upper bound is the inputted gas parameter. */ - private Bytes estimateGas(final ContractExecutionParameters params) { - final var processingResult = doProcessCall(params, params.getGas(), true); + private Bytes estimateGas(final ContractExecutionParameters params, final ContractCallContext context) { + final var processingResult = callContract(params, context); validateResult(processingResult, CallType.ETH_ESTIMATE_GAS); final var gasUsedByInitialCall = processingResult.getGasUsed(); diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/TransactionExecutionService.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/TransactionExecutionService.java index c83dc9b118d..5d8dc059c10 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/TransactionExecutionService.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/TransactionExecutionService.java @@ -35,30 +35,25 @@ import com.hedera.hapi.node.transaction.TransactionBody; import com.hedera.hapi.node.transaction.TransactionRecord; import com.hedera.mirror.web3.common.ContractCallContext; +import com.hedera.mirror.web3.evm.contracts.execution.traceability.MirrorOperationTracer; import com.hedera.mirror.web3.evm.contracts.execution.traceability.OpcodeTracer; import com.hedera.mirror.web3.evm.properties.MirrorNodeEvmProperties; import com.hedera.mirror.web3.exception.MirrorEvmTransactionException; import com.hedera.mirror.web3.service.model.CallServiceParameters; import com.hedera.mirror.web3.service.model.CallServiceParameters.CallType; import com.hedera.mirror.web3.state.AliasesReadableKVState; -import com.hedera.node.app.config.ConfigProviderImpl; import com.hedera.node.app.service.evm.contracts.execution.HederaEvmTransactionProcessingResult; import com.hedera.node.app.service.token.TokenService; -import com.hedera.node.app.workflows.standalone.TransactionExecutor; -import com.hedera.node.app.workflows.standalone.TransactionExecutors; -import com.hedera.node.app.workflows.standalone.TransactionExecutors.TracerBinding; import com.hedera.node.config.data.EntitiesConfig; -import com.swirlds.config.api.Configuration; import com.swirlds.state.State; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Meter.MeterProvider; -import jakarta.annotation.Nullable; import jakarta.inject.Named; import java.time.Instant; import java.util.List; -import java.util.Map; import java.util.Optional; import lombok.CustomLog; +import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.apache.tuweni.bytes.Bytes; import org.hyperledger.besu.datatypes.Address; @@ -66,32 +61,27 @@ @Named @CustomLog +@RequiredArgsConstructor public class TransactionExecutionService { - private static final Configuration DEFAULT_CONFIG = new ConfigProviderImpl().getConfiguration(); - private static final OperationTracer[] EMPTY_OPERATION_TRACER_ARRAY = new OperationTracer[0]; private static final AccountID TREASURY_ACCOUNT_ID = AccountID.newBuilder().accountNum(2).build(); private static final Duration TRANSACTION_DURATION = new Duration(15); private static final int INITCODE_SIZE_KB = 6 * 1024; + private final State mirrorNodeState; private final MirrorNodeEvmProperties mirrorNodeEvmProperties; private final OpcodeTracer opcodeTracer; - - protected TransactionExecutionService( - State mirrorNodeState, MirrorNodeEvmProperties mirrorNodeEvmProperties, OpcodeTracer opcodeTracer) { - this.mirrorNodeState = mirrorNodeState; - this.mirrorNodeEvmProperties = mirrorNodeEvmProperties; - this.opcodeTracer = opcodeTracer; - } + private final MirrorOperationTracer mirrorOperationTracer; + private final TransactionExecutorFactory transactionExecutorFactory; public HederaEvmTransactionProcessingResult execute( final CallServiceParameters params, final long estimatedGas, final MeterProvider gasUsedCounter) { final var isContractCreate = params.getReceiver().isZero(); + final var configuration = mirrorNodeEvmProperties.getVersionedConfiguration(); final var maxLifetime = - DEFAULT_CONFIG.getConfigData(EntitiesConfig.class).maxLifetime(); - var executor = - ExecutorFactory.newExecutor(mirrorNodeState, mirrorNodeEvmProperties.getTransactionProperties(), null); + configuration.getConfigData(EntitiesConfig.class).maxLifetime(); + var executor = transactionExecutorFactory.get(); TransactionBody transactionBody; HederaEvmTransactionProcessingResult result = null; @@ -243,6 +233,7 @@ private TransactionBody buildContractCallTransactionBody( .build()) .functionParameters(com.hedera.pbj.runtime.io.buffer.Bytes.wrap( params.getCallData().toArrayUnsafe())) + .amount(params.getValue()) // tinybars sent to contract .gas(estimatedGas) .build()) .build(); @@ -275,19 +266,6 @@ private AccountID getSenderAccountID(final CallServiceParameters params) { private OperationTracer[] getOperationTracers() { return ContractCallContext.get().getOpcodeTracerOptions() != null ? new OperationTracer[] {opcodeTracer} - : EMPTY_OPERATION_TRACER_ARRAY; - } - - public static class ExecutorFactory { - - private ExecutorFactory() {} - - public static TransactionExecutor newExecutor( - State mirrorNodeState, - Map properties, - @Nullable final TracerBinding customTracerBinding) { - return TransactionExecutors.TRANSACTION_EXECUTORS.newExecutor( - mirrorNodeState, properties, customTracerBinding); - } + : new OperationTracer[] {mirrorOperationTracer}; } } diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/TransactionExecutorFactory.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/TransactionExecutorFactory.java new file mode 100644 index 00000000000..0b7a06c477e --- /dev/null +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/service/TransactionExecutorFactory.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.web3.service; + +import com.hedera.hapi.node.base.SemanticVersion; +import com.hedera.mirror.web3.evm.properties.MirrorNodeEvmProperties; +import com.hedera.node.app.workflows.standalone.TransactionExecutor; +import com.hedera.node.app.workflows.standalone.TransactionExecutors; +import com.swirlds.state.State; +import jakarta.inject.Named; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import lombok.RequiredArgsConstructor; + +@Named +@RequiredArgsConstructor +public class TransactionExecutorFactory { + + private final State mirrorNodeState; + private final MirrorNodeEvmProperties mirrorNodeEvmProperties; + private final Map transactionExecutors = new ConcurrentHashMap<>(); + + // Reuse TransactionExecutor across requests for the same EVM version + public TransactionExecutor get() { + var version = mirrorNodeEvmProperties.getSemanticEvmVersion(); + return transactionExecutors.computeIfAbsent(version, this::create); + } + + private TransactionExecutor create(SemanticVersion evmVersion) { + var properties = new HashMap<>(mirrorNodeEvmProperties.getTransactionProperties()); + properties.put("contracts.evm.version", "v" + evmVersion.major() + "." + evmVersion.minor()); + return TransactionExecutors.TRANSACTION_EXECUTORS.newExecutor(mirrorNodeState, properties, null); + } +} diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/MirrorNodeState.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/MirrorNodeState.java index ef8d45acced..7eec88878a9 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/MirrorNodeState.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/MirrorNodeState.java @@ -35,8 +35,8 @@ import com.hedera.mirror.web3.state.core.MapReadableStates; import com.hedera.mirror.web3.state.core.MapWritableKVState; import com.hedera.mirror.web3.state.core.MapWritableStates; +import com.hedera.mirror.web3.state.singleton.SingletonState; import com.hedera.node.app.config.BootstrapConfigProviderImpl; -import com.hedera.node.app.config.ConfigProviderImpl; import com.hedera.node.app.fees.FeeService; import com.hedera.node.app.ids.EntityIdService; import com.hedera.node.app.records.BlockRecordService; @@ -84,7 +84,6 @@ import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import lombok.RequiredArgsConstructor; @@ -125,8 +124,8 @@ private void init() { null, new ServicesSoftwareVersion( bootstrapConfig.getConfigData(VersionConfig.class).servicesVersion()), - new ConfigProviderImpl().getConfiguration(), - new ConfigProviderImpl().getConfiguration(), + mirrorNodeEvmProperties.getVersionedConfiguration(), + mirrorNodeEvmProperties.getVersionedConfiguration(), networkInfo, UnavailableMetrics.UNAVAILABLE_METRICS, startupNetworks); @@ -192,8 +191,8 @@ public ReadableStates getReadableStates(@Nonnull String serviceName) { } else { data.put(stateName, new MapReadableKVState(stateName, map)); } - } else if (state instanceof AtomicReference ref) { - data.put(stateName, new ReadableSingletonStateBase<>(stateName, ref::get)); + } else if (state instanceof SingletonState singleton) { + data.put(stateName, new ReadableSingletonStateBase<>(stateName, singleton)); } } return new MapReadableStates(data); @@ -224,7 +223,7 @@ public WritableStates getWritableStates(@Nonnull String serviceName) { new MapWritableKVState<>( stateName, getReadableStates(serviceName).get(stateName)))); - } else if (state instanceof AtomicReference ref) { + } else if (state instanceof SingletonState ref) { data.put(stateName, withAnyRegisteredListeners(serviceName, stateName, ref)); } } @@ -253,8 +252,10 @@ public void commit() { } private WritableSingletonStateBase withAnyRegisteredListeners( - @Nonnull final String serviceName, @Nonnull final String stateKey, @Nonnull final AtomicReference ref) { - final var state = new WritableSingletonStateBase<>(stateKey, ref::get, ref::set); + @Nonnull final String serviceName, + @Nonnull final String stateKey, + @Nonnull final SingletonState singleton) { + final var state = new WritableSingletonStateBase<>(stateKey, singleton, singleton::set); listeners.forEach(listener -> { if (listener.stateTypes().contains(SINGLETON)) { registerSingletonListener(serviceName, state, listener); @@ -361,11 +362,11 @@ private void registerServices(ServicesRegistry servicesRegistry) { InstantSource.system(), signatureVerifier(), Gossip.UNAVAILABLE_GOSSIP, - () -> new ConfigProviderImpl().getConfiguration(), + () -> mirrorNodeEvmProperties.getVersionedConfiguration(), () -> DEFAULT_NODE_INFO, () -> UNAVAILABLE_METRICS, new AppThrottleFactory( - () -> new ConfigProviderImpl().getConfiguration(), + () -> mirrorNodeEvmProperties.getVersionedConfiguration(), () -> this, () -> ThrottleDefinitions.DEFAULT, ThrottleAccumulator::new)); diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/components/SchemaRegistryImpl.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/components/SchemaRegistryImpl.java index 9504a34deb6..f35a80676b7 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/components/SchemaRegistryImpl.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/components/SchemaRegistryImpl.java @@ -23,6 +23,8 @@ import com.hedera.hapi.node.base.SemanticVersion; import com.hedera.mirror.web3.state.MirrorNodeState; import com.hedera.mirror.web3.state.core.MapWritableStates; +import com.hedera.mirror.web3.state.singleton.DefaultSingleton; +import com.hedera.mirror.web3.state.singleton.SingletonState; import com.hedera.node.app.state.merkle.SchemaApplications; import com.swirlds.config.api.Configuration; import com.swirlds.config.api.ConfigurationBuilder; @@ -37,6 +39,7 @@ import com.swirlds.state.spi.WritableStates; import jakarta.annotation.Nonnull; import jakarta.annotation.Nullable; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -45,7 +48,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -54,6 +58,7 @@ public class SchemaRegistryImpl implements SchemaRegistry { public static final SemanticVersion CURRENT_VERSION = new SemanticVersion(0, 47, 0, "SNAPSHOT", ""); + private final Collection> singletons; private final SchemaApplications schemaApplications; /** @@ -227,9 +232,11 @@ private RedefinedWritableStates applyStateDefinitions( @Nonnull final Configuration configuration, @Nonnull final MirrorNodeState state) { final Map stateDataSources = new HashMap<>(); + var singletonMap = singletons.stream().collect(Collectors.toMap(SingletonState::getKey, Function.identity())); schema.statesToCreate(configuration).forEach(def -> { if (def.singleton()) { - stateDataSources.put(def.stateKey(), new AtomicReference<>()); + var singleton = singletonMap.computeIfAbsent(def.stateKey(), DefaultSingleton::new); + stateDataSources.put(singleton.getKey(), singleton); } else if (def.queue()) { stateDataSources.put(def.stateKey(), new ConcurrentLinkedDeque<>()); } else { diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/components/ServicesRegistryImpl.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/components/ServicesRegistryImpl.java index a4ca5025a35..ebab724bbe9 100644 --- a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/components/ServicesRegistryImpl.java +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/components/ServicesRegistryImpl.java @@ -16,20 +16,25 @@ package com.hedera.mirror.web3.state.components; +import com.hedera.mirror.web3.state.singleton.SingletonState; import com.hedera.node.app.services.ServicesRegistry; import com.hedera.node.app.state.merkle.SchemaApplications; import com.swirlds.state.lifecycle.Service; import jakarta.annotation.Nonnull; import jakarta.inject.Named; +import java.util.Collection; import java.util.Collections; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import lombok.RequiredArgsConstructor; @Named +@RequiredArgsConstructor public class ServicesRegistryImpl implements ServicesRegistry { private final SortedSet entries = new TreeSet<>(); + private final Collection> singletons; @Nonnull @Override @@ -39,7 +44,7 @@ public Set registrations() { @Override public void register(@Nonnull Service service) { - final var registry = new SchemaRegistryImpl(new SchemaApplications()); + final var registry = new SchemaRegistryImpl(singletons, new SchemaApplications()); service.registerSchemas(registry); entries.add(new ServicesRegistryImpl.Registration(service, registry)); } @@ -48,7 +53,7 @@ public void register(@Nonnull Service service) { @Override public ServicesRegistry subRegistryFor(@Nonnull String... serviceNames) { final var selections = Set.of(serviceNames); - final var subRegistry = new ServicesRegistryImpl(); + final var subRegistry = new ServicesRegistryImpl(singletons); subRegistry.entries.addAll(entries.stream() .filter(registration -> selections.contains(registration.serviceName())) .toList()); diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/BlockInfoSingleton.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/BlockInfoSingleton.java new file mode 100644 index 00000000000..648499f1e54 --- /dev/null +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/BlockInfoSingleton.java @@ -0,0 +1,93 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.web3.state.singleton; + +import static com.hedera.node.app.records.schemas.V0490BlockRecordSchema.BLOCK_INFO_STATE_KEY; + +import com.hedera.hapi.node.state.blockrecords.BlockInfo; +import com.hedera.mirror.common.domain.transaction.RecordFile; +import com.hedera.mirror.web3.common.ContractCallContext; +import com.hedera.mirror.web3.evm.properties.MirrorNodeEvmProperties; +import com.hedera.mirror.web3.repository.RecordFileRepository; +import com.hedera.mirror.web3.state.Utils; +import com.hedera.node.config.data.BlockRecordStreamConfig; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import jakarta.inject.Named; +import java.nio.ByteBuffer; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import org.apache.commons.codec.binary.Hex; + +@Named +@RequiredArgsConstructor +public class BlockInfoSingleton implements SingletonState { + + private static final int HASH_SIZE = 48; + + private final MirrorNodeEvmProperties properties; + private final RecordFileRepository recordFileRepository; + + @Override + public String getKey() { + return BLOCK_INFO_STATE_KEY; + } + + @Override + public BlockInfo get() { + var recordFile = ContractCallContext.get().getRecordFile(); + var blockHashes = getBlockHashes(recordFile); + var startTimestamp = Utils.convertToTimestamp(recordFile.getConsensusStart()); + var endTimestamp = Utils.convertToTimestamp(recordFile.getConsensusEnd()); + + return BlockInfo.newBuilder() + .blockHashes(Bytes.wrap(blockHashes)) + .consTimeOfLastHandledTxn(endTimestamp) + .firstConsTimeOfCurrentBlock(endTimestamp) + .firstConsTimeOfLastBlock(startTimestamp) + .lastBlockNumber(recordFile.getIndex()) + .migrationRecordsStreamed(true) + .build(); + } + + /** + * Loads the last 256 block hashes from the database into a single byte array. This is inefficient to do for every + * request when only a few requests may have a BLOCKHASH operation. This method is temporary until the EVM library + * supports custom implementations for operations that don't need to use the BlockInfo singleton. + */ + @SneakyThrows + private byte[] getBlockHashes(RecordFile recordFile) { + if (recordFile.getIndex() == 0) { + return Hex.decodeHex(recordFile.getHash()); + } + + var config = properties.getVersionedConfiguration().getConfigData(BlockRecordStreamConfig.class); + int blockHashCount = config.numOfBlockHashesInState(); + long endIndex = recordFile.getIndex() - 1; // Optimization: Don't reload the record file from the context + long startIndex = Math.max(0L, endIndex - blockHashCount + 1); + + var blocks = recordFileRepository.findByIndexRange(startIndex, endIndex); + blocks.add(recordFile); + var buffer = ByteBuffer.allocate(blocks.size() * HASH_SIZE); + + for (var block : blocks) { + var hash = Hex.decodeHex(block.getHash()); + buffer.put(hash); + } + + return buffer.array(); + } +} diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/DefaultSingleton.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/DefaultSingleton.java new file mode 100644 index 00000000000..6f8aad8d367 --- /dev/null +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/DefaultSingleton.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.web3.state.singleton; + +import java.util.concurrent.atomic.AtomicReference; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public class DefaultSingleton extends AtomicReference implements SingletonState { + + private final String key; +} diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/RunningHashesSingleton.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/RunningHashesSingleton.java new file mode 100644 index 00000000000..2b7c1c6cec4 --- /dev/null +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/RunningHashesSingleton.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.web3.state.singleton; + +import static com.hedera.node.app.records.schemas.V0490BlockRecordSchema.RUNNING_HASHES_STATE_KEY; + +import com.hedera.hapi.node.state.blockrecords.RunningHashes; +import com.hedera.mirror.web3.common.ContractCallContext; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import jakarta.inject.Named; +import lombok.RequiredArgsConstructor; + +@Named +@RequiredArgsConstructor +public class RunningHashesSingleton implements SingletonState { + + @Override + public String getKey() { + return RUNNING_HASHES_STATE_KEY; + } + + @Override + public RunningHashes get() { + var recordFile = ContractCallContext.get().getRecordFile(); + return RunningHashes.newBuilder() + .runningHash(Bytes.fromHex(recordFile.getHash())) + .build(); + } +} diff --git a/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/SingletonState.java b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/SingletonState.java new file mode 100644 index 00000000000..b909db79e4a --- /dev/null +++ b/hedera-mirror-web3/src/main/java/com/hedera/mirror/web3/state/singleton/SingletonState.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.web3.state.singleton; + +import java.util.function.Supplier; + +public interface SingletonState extends Supplier { + + String getKey(); + + default void set(T value) { + // Do nothing since our singletons are either immutable static data or dynamically retrieved from the db. + } +} diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/common/ContractCallContextTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/common/ContractCallContextTest.java index f4801db9dd2..8459fc89965 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/common/ContractCallContextTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/common/ContractCallContextTest.java @@ -18,7 +18,6 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.hedera.mirror.common.domain.DomainBuilder; import com.hedera.mirror.common.domain.transaction.RecordFile; import com.hedera.mirror.web3.ContextExtension; import com.hedera.mirror.web3.evm.store.StackedStateFrames; @@ -30,14 +29,8 @@ @ExtendWith(ContextExtension.class) class ContractCallContextTest { - private final StackedStateFrames stackedStateFrames; - - private final DomainBuilder domainBuilder = new DomainBuilder(); - - public ContractCallContextTest() { - stackedStateFrames = new StackedStateFrames(List.of( - new BareDatabaseAccessor() {}, new BareDatabaseAccessor() {})); - } + private final StackedStateFrames stackedStateFrames = new StackedStateFrames( + List.of(new BareDatabaseAccessor() {}, new BareDatabaseAccessor() {})); @Test void testGet() { @@ -45,17 +38,6 @@ void testGet() { assertThat(ContractCallContext.get()).isEqualTo(context); } - @Test - void testRecordFileIsClearedOnReset() { - var context = ContractCallContext.get(); - final var recordFile = domainBuilder.recordFile().get(); - context.setRecordFile(recordFile); - assertThat(context.getRecordFile()).isEqualTo(recordFile); - - context.reset(); - assertThat(context.getRecordFile()).isNull(); - } - @Test void testReset() { var context = ContractCallContext.get(); @@ -65,7 +47,6 @@ void testReset() { context.setStack(stackedStateFrames.top()); context.reset(); - assertThat(context.getRecordFile()).isNull(); assertThat(context.getStack()).isEqualTo(context.getStackBase()); } } diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/repository/RecordFileRepositoryTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/repository/RecordFileRepositoryTest.java index b2a8f3df6c6..b4b84dadb83 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/repository/RecordFileRepositoryTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/repository/RecordFileRepositoryTest.java @@ -48,16 +48,7 @@ void findEarliest() { } @Test - void findFileHashByIndex() { - final var file = domainBuilder.recordFile().persist(); - - assertThat(recordFileRepository.findByIndex(file.getIndex())) - .map(RecordFile::getHash) - .hasValue(file.getHash()); - } - - @Test - void findLatestFile() { + void findLatest() { domainBuilder.recordFile().persist(); var latest = domainBuilder.recordFile().persist(); @@ -65,7 +56,7 @@ void findLatestFile() { } @Test - void findRecordFileByIndex() { + void findByIndex() { domainBuilder.recordFile().persist(); var latest = domainBuilder.recordFile().persist(); long blockNumber = latest.getIndex(); @@ -76,13 +67,23 @@ void findRecordFileByIndex() { } @Test - void findRecordFileByIndexNotExists() { + void findByIndexNotExists() { long nonExistentBlockNumber = 1L; assertThat(recordFileRepository.findByIndex(nonExistentBlockNumber)).isEmpty(); } @Test - void findRecordFileByTimestamp() { + void findByIndexRange() { + domainBuilder.recordFile().persist(); + var recordFile2 = domainBuilder.recordFile().persist(); + var recordFile3 = domainBuilder.recordFile().persist(); + domainBuilder.recordFile().persist(); + assertThat(recordFileRepository.findByIndexRange(recordFile2.getIndex(), recordFile3.getIndex())) + .containsExactly(recordFile2, recordFile3); + } + + @Test + void findByTimestamp() { var timestamp = domainBuilder.timestamp(); var recordFile = domainBuilder .recordFile() diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/AbstractContractCallServiceOpcodeTracerTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/AbstractContractCallServiceOpcodeTracerTest.java index 846368caa75..dff68e529b3 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/AbstractContractCallServiceOpcodeTracerTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/AbstractContractCallServiceOpcodeTracerTest.java @@ -25,6 +25,7 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.mockito.Mockito.doAnswer; +import com.hedera.mirror.common.domain.balance.AccountBalance; import com.hedera.mirror.common.domain.entity.Entity; import com.hedera.mirror.common.domain.entity.EntityId; import com.hedera.mirror.rest.model.OpcodesResponse; @@ -237,4 +238,18 @@ private Address entityAddress(Entity entity) { } return toAddress(entity.toEntityId()); } + + protected void accountBalanceRecordsPersist(Entity sender) { + domainBuilder + .accountBalance() + .customize(ab -> ab.id(new AccountBalance.Id(sender.getCreatedTimestamp(), sender.toEntityId())) + .balance(sender.getBalance())) + .persist(); + + domainBuilder + .accountBalance() + .customize(ab -> ab.id(new AccountBalance.Id(sender.getCreatedTimestamp(), treasuryEntity.toEntityId())) + .balance(treasuryEntity.getBalance())) + .persist(); + } } diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/AbstractContractCallServiceTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/AbstractContractCallServiceTest.java index 209996f6ca1..2c92cd17686 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/AbstractContractCallServiceTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/AbstractContractCallServiceTest.java @@ -16,6 +16,7 @@ package com.hedera.mirror.web3.service; +import static com.hedera.mirror.web3.evm.pricing.RatesAndFeesLoader.DEFAULT_FEE_SCHEDULE; import static com.hedera.mirror.web3.utils.ContractCallTestUtil.ESTIMATE_GAS_ERROR_MESSAGE; import static com.hedera.mirror.web3.utils.ContractCallTestUtil.TRANSACTION_GAS_LIMIT; import static com.hedera.mirror.web3.utils.ContractCallTestUtil.isWithinExpectedGasRange; @@ -25,6 +26,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import com.hedera.mirror.common.domain.balance.AccountBalance; +import com.hedera.mirror.common.domain.balance.TokenBalance; import com.hedera.mirror.common.domain.entity.Entity; import com.hedera.mirror.common.domain.entity.EntityId; import com.hedera.mirror.common.domain.entity.EntityType; @@ -32,9 +34,9 @@ import com.hedera.mirror.common.domain.token.TokenKycStatusEnum; import com.hedera.mirror.common.domain.transaction.RecordFile; import com.hedera.mirror.web3.Web3IntegrationTest; -import com.hedera.mirror.web3.common.ContractCallContext; import com.hedera.mirror.web3.evm.properties.MirrorNodeEvmProperties; import com.hedera.mirror.web3.evm.utils.EvmTokenUtils; +import com.hedera.mirror.web3.exception.MirrorEvmTransactionException; import com.hedera.mirror.web3.service.model.CallServiceParameters.CallType; import com.hedera.mirror.web3.service.model.ContractDebugParameters; import com.hedera.mirror.web3.service.model.ContractExecutionParameters; @@ -89,8 +91,10 @@ public abstract class AbstractContractCallServiceTest extends Web3IntegrationTes @Resource protected State state; - protected RecordFile genesisRecordFile; + @Resource + protected ContractExecutionService contractExecutionService; + protected RecordFile genesisRecordFile; protected Entity treasuryEntity; public static Key getKeyWithDelegatableContractId(final Contract contract) { @@ -111,8 +115,11 @@ public static Key getKeyWithContractId(final Contract contract) { @BeforeEach protected void setup() { - genesisRecordFile = - domainBuilder.recordFile().customize(f -> f.index(0L)).persist(); + // Change this to not be epoch once services fixes config updates for non-genesis flow + genesisRecordFile = domainBuilder + .recordFile() + .customize(f -> f.consensusEnd(0L).consensusStart(0L).index(0L)) + .persist(); treasuryEntity = domainBuilder .entity() .customize(e -> e.id(2L).num(2L).balance(5000000000000000000L)) @@ -128,6 +135,10 @@ protected void setup() { treasuryEntity.getCreatedTimestamp(), treasuryEntity.toEntityId())) .balance(treasuryEntity.getBalance())) .persist(); + domainBuilder + .fileData() + .customize(f -> f.entityId(EntityId.of(111L)).fileData(DEFAULT_FEE_SCHEDULE.toByteArray())) + .persist(); testWeb3jService.reset(); } @@ -136,17 +147,19 @@ void cleanup() { testWeb3jService.reset(); } - @SuppressWarnings("try") protected long gasUsedAfterExecution(final ContractExecutionParameters serviceParameters) { - return ContractCallContext.run(ctx -> { - ctx.initializeStackFrames(store.getStackedStateFrames()); - long result = processor - .execute(serviceParameters, serviceParameters.getGas()) - .getGasUsed(); - - assertThat(store.getStackedStateFrames().height()).isEqualTo(1); - return result; - }); + try { + return contractExecutionService.callContract(serviceParameters).getGasUsed(); + } catch (MirrorEvmTransactionException e) { + var result = e.getResult(); + + // Some tests expect to fail but still want to capture the gas used + if (result != null) { + return result.getGasUsed(); + } + + throw e; + } } protected void verifyEthCallAndEstimateGas( @@ -235,7 +248,7 @@ protected Entity accountEntityPersist() { return domainBuilder .entity() .customize(e -> - e.type(EntityType.ACCOUNT).evmAddress(null).alias(null).balance(1_000_000_000_000L)) + e.type(EntityType.ACCOUNT).evmAddress(null).alias(null).balance(100_000_000_000_000_000L)) .persist(); } @@ -261,6 +274,30 @@ protected void tokenAccountPersist(final Entity token, final Long accountId) { .persist(); } + protected void persistAccountBalance(Entity account, long balance, long timestamp) { + domainBuilder + .accountBalance() + .customize(ab -> ab.id(new AccountBalance.Id(timestamp, account.toEntityId())) + .balance(balance)) + .persist(); + } + + protected void persistAccountBalance(Entity account, long balance) { + domainBuilder + .accountBalance() + .customize(ab -> ab.id(new AccountBalance.Id(account.getCreatedTimestamp(), account.toEntityId())) + .balance(balance)) + .persist(); + } + + protected void persistTokenBalance(Entity account, Entity token, long timestamp) { + domainBuilder + .tokenBalance() + .customize(ab -> ab.id(new TokenBalance.Id(timestamp, account.toEntityId(), token.toEntityId())) + .balance(100)) + .persist(); + } + protected String getAddressFromEntity(Entity entity) { return EvmTokenUtils.toAddress(entity.toEntityId()).toHexString(); } diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallAddressThisTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallAddressThisTest.java index 7f687369382..dae977b1667 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallAddressThisTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallAddressThisTest.java @@ -20,11 +20,10 @@ import static com.hedera.mirror.common.util.DomainUtils.toEvmAddress; import static com.hedera.mirror.web3.evm.utils.EvmTokenUtils.entityIdFromEvmAddress; import static com.hedera.mirror.web3.service.model.CallServiceParameters.CallType.ETH_ESTIMATE_GAS; +import static com.hedera.mirror.web3.utils.ContractCallTestUtil.ESTIMATE_GAS_ERROR_MESSAGE; import static com.hedera.mirror.web3.utils.ContractCallTestUtil.isWithinExpectedGasRange; import static com.hedera.mirror.web3.utils.ContractCallTestUtil.longValueOf; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -34,18 +33,13 @@ import com.hedera.mirror.web3.viewmodel.ContractCallRequest; import com.hedera.mirror.web3.web3j.generated.TestAddressThis; import com.hedera.mirror.web3.web3j.generated.TestNestedAddressThis; -import com.hedera.node.app.service.evm.contracts.execution.HederaEvmTransactionProcessingResult; import jakarta.annotation.Resource; import java.math.BigInteger; -import java.util.ArrayList; -import java.util.List; import lombok.SneakyThrows; -import org.apache.tuweni.bytes.Bytes; import org.hyperledger.besu.datatypes.Address; import org.junit.jupiter.api.Test; import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; import org.springframework.http.MediaType; -import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.springframework.test.web.servlet.MockMvc; import org.springframework.test.web.servlet.ResultActions; import org.testcontainers.shaded.org.apache.commons.lang3.StringUtils; @@ -64,9 +58,6 @@ class ContractCallAddressThisTest extends AbstractContractCallServiceTest { @Resource private ObjectMapper objectMapper; - @MockitoSpyBean - private ContractExecutionService contractExecutionService; - @SneakyThrows private ResultActions contractCall(ContractCallRequest request) { return mockMvc.perform(post(CALL_URI) @@ -81,8 +72,9 @@ void deployAddressThisContract() { final var serviceParameters = testWeb3jService.serviceParametersForTopLevelContractCreate( contract.getContractBinary(), ETH_ESTIMATE_GAS, Address.ZERO); final long actualGas = 57764L; - assertThat(isWithinExpectedGasRange( - longValueOf.applyAsLong(contractCallService.processCall(serviceParameters)), actualGas)) + long expectedGas = longValueOf.applyAsLong(contractCallService.processCall(serviceParameters)); + assertThat(isWithinExpectedGasRange(expectedGas, actualGas)) + .withFailMessage(ESTIMATE_GAS_ERROR_MESSAGE, expectedGas, actualGas) .isTrue(); } @@ -100,22 +92,16 @@ void addressThisEthCallWithoutEvmAlias() throws Exception { testWeb3jService.deployWithoutPersistWithValue(TestAddressThis::deploy, BigInteger.valueOf(1000)); addressThisContractPersist( testWeb3jService.getContractRuntime(), Address.fromHexString(contract.getContractAddress())); - final List capturedOutputs = new ArrayList<>(); - doAnswer(invocation -> { - HederaEvmTransactionProcessingResult result = - (HederaEvmTransactionProcessingResult) invocation.callRealMethod(); - capturedOutputs.add(result.getOutput()); // Capture the result - return result; - }) - .when(contractExecutionService) - .callContract(any(), any()); // When - final var result = contract.call_getAddressThis().send(); + var callFunction = contract.call_getAddressThis(); + final var result = callFunction.send(); + var parameters = getContractExecutionParameters(callFunction, contract); + var output = contractExecutionService.callContract(parameters).getOutput(); // Then final var successfulResponse = "0x" + StringUtils.leftPad(result.substring(2), 64, '0'); - assertThat(successfulResponse).isEqualTo(capturedOutputs.getFirst().toHexString()); + assertThat(successfulResponse).isEqualTo(output.toHexString()); } @Test @@ -161,8 +147,9 @@ void deployNestedAddressThisContract() { final var serviceParameters = testWeb3jService.serviceParametersForTopLevelContractCreate( contract.getContractBinary(), ETH_ESTIMATE_GAS, Address.ZERO); final long actualGas = 95401L; - assertThat(isWithinExpectedGasRange( - longValueOf.applyAsLong(contractCallService.processCall(serviceParameters)), actualGas)) + long expectedGas = longValueOf.applyAsLong(contractCallService.processCall(serviceParameters)); + assertThat(isWithinExpectedGasRange(expectedGas, actualGas)) + .withFailMessage(ESTIMATE_GAS_ERROR_MESSAGE, expectedGas, actualGas) .isTrue(); } diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallEvmCodesTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallEvmCodesTest.java index d46df952639..f5f755a61bd 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallEvmCodesTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallEvmCodesTest.java @@ -166,32 +166,34 @@ void getBlockPrevrandao() throws Exception { @Test void getBlockHashReturnsCorrectHash() throws Exception { testWeb3jService.setUseContractCallDeploy(true); - domainBuilder.recordFile().persist(); // Set a latest block - needed for the block hash operation + domainBuilder.recordFile().customize(r -> r.index(1L)).persist(); + var latest = domainBuilder.recordFile().customize(r -> r.index(2L)).persist(); final var contract = testWeb3jService.deploy(EvmCodes::deploy); - var result = contract.call_getBlockHash(BigInteger.valueOf(genesisRecordFile.getIndex())) + var result = contract.call_getBlockHash(BigInteger.valueOf(latest.getIndex())) .send(); - var expectedResult = - ByteString.fromHex(genesisRecordFile.getHash().substring(0, 64)).toByteArray(); + var expectedResult = Hex.decode(latest.getHash().substring(0, 64)); assertThat(result).isEqualTo(expectedResult); } @Test void getGenesisBlockHashReturnsCorrectBlock() throws Exception { testWeb3jService.setUseContractCallDeploy(true); - domainBuilder.recordFile().persist(); // Set a latest block - needed for the block hash operation + domainBuilder.recordFile().customize(r -> r.index(1L)).persist(); + domainBuilder.recordFile().customize(r -> r.index(2L)).persist(); + final var contract = testWeb3jService.deploy(EvmCodes::deploy); var result = contract.call_getBlockHash(BigInteger.ZERO).send(); - var expectedResult = - ByteString.fromHex(genesisRecordFile.getHash().substring(0, 64)).toByteArray(); + + var expectedResult = Hex.decode(genesisRecordFile.getHash().substring(0, 64)); assertThat(result).isEqualTo(expectedResult); } @Test void getLatestBlockHashIsNotEmpty() throws Exception { - domainBuilder.recordFile().persist(); + domainBuilder.recordFile().customize(r -> r.index(1L)).persist(); final var contract = testWeb3jService.deploy(EvmCodes::deploy); var result = contract.call_getLatestBlockHash().send(); - var expectedResult = ByteString.fromHex((EMPTY_BLOCK_HASH)).toByteArray(); + var expectedResult = Hex.decode(EMPTY_BLOCK_HASH); assertThat(result).isNotEqualTo(expectedResult); } @@ -200,7 +202,7 @@ void getBlockHashAfterTheLatestReturnsZero() throws Exception { final var contract = testWeb3jService.deploy(EvmCodes::deploy); var result = contract.call_getBlockHash(BigInteger.valueOf(Long.MAX_VALUE)).send(); - var expectedResult = ByteString.fromHex((EMPTY_BLOCK_HASH)).toByteArray(); + var expectedResult = Hex.decode(EMPTY_BLOCK_HASH); assertThat(result).isEqualTo(expectedResult); } diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallNativePrecompileTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallNativePrecompileTest.java index 95d6b6bbe29..c87af8bb724 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallNativePrecompileTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallNativePrecompileTest.java @@ -16,6 +16,7 @@ package com.hedera.mirror.web3.service; +import static com.hedera.mirror.web3.evm.pricing.RatesAndFeesLoader.DEFAULT_FEE_SCHEDULE; import static com.hedera.mirror.web3.service.AbstractContractCallServiceTest.EXCHANGE_RATES_SET; import static com.hedera.mirror.web3.service.ContractExecutionService.GAS_USED_METRIC; import static com.hedera.mirror.web3.service.model.CallServiceParameters.CallType; @@ -49,6 +50,10 @@ void setup() { .fileData() .customize(f -> f.entityId(EntityId.of(112L)).fileData(EXCHANGE_RATES_SET)) .persist(); + domainBuilder + .fileData() + .customize(f -> f.entityId(EntityId.of(111L)).fileData(DEFAULT_FEE_SCHEDULE.toByteArray())) + .persist(); } @Test diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallServicePrecompileModificationTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallServicePrecompileModificationTest.java index 4813fac066b..b17fdf72874 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallServicePrecompileModificationTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallServicePrecompileModificationTest.java @@ -60,7 +60,6 @@ import com.hedera.services.store.models.Id; import com.hedera.services.utils.EntityIdUtils; import com.hederahashgraph.api.proto.java.Key.KeyCase; -import com.swirlds.base.time.Time; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.time.Instant; @@ -665,26 +664,32 @@ void unpauseToken() throws Exception { @Test void createFungibleToken() throws Exception { // Given - var initialSupply = BigInteger.valueOf(10L); - var decimals = BigInteger.valueOf(10L); - var value = 10000L * 100_000_000L; + final var value = 10000L * 100_000_000_000L; + final var sender = accountEntityPersist(); + + accountBalanceRecordsPersist(sender); final var contract = testWeb3jService.deploy(ModificationPrecompileTestContract::deploy); - final var treasuryAccount = domainBuilder - .entity() - .customize(e -> e.type(EntityType.ACCOUNT).deleted(false).evmAddress(null)) - .persist(); + testWeb3jService.setValue(value); + + testWeb3jService.setSender(toAddress(sender.toEntityId()).toHexString()); + + final var treasuryAccount = accountEntityPersist(); + final var token = populateHederaToken( contract.getContractAddress(), TokenTypeEnum.FUNGIBLE_COMMON, treasuryAccount.toEntityId()); + final var initialTokenSupply = BigInteger.valueOf(10L); + final var decimalPlacesSupportedByToken = BigInteger.valueOf(10L); // e.g. 1.0123456789 // When - testWeb3jService.setValue(value); - final var functionCall = contract.call_createFungibleTokenExternal(token, initialSupply, decimals); + final var functionCall = + contract.call_createFungibleTokenExternal(token, initialTokenSupply, decimalPlacesSupportedByToken); final var result = functionCall.send(); final var contractFunctionProvider = ContractFunctionProviderRecord.builder() .contractAddress(Address.fromHexString(contract.getContractAddress())) + .sender(toAddress(sender.toEntityId())) .value(value) .build(); @@ -702,15 +707,22 @@ void createFungibleTokenWithCustomFees() throws Exception { var decimals = BigInteger.valueOf(10L); var value = 10000L * 100_000_000L; + final var sender = accountEntityPersist(); + + accountBalanceRecordsPersist(sender); + + final var treasuryAccount = accountEntityPersist(); + final var tokenForDenomination = persistFungibleToken(); final var feeCollector = accountEntityWithEvmAddressPersist(); + tokenAccountPersist(tokenForDenomination, feeCollector); + final var contract = testWeb3jService.deploy(ModificationPrecompileTestContract::deploy); - final var treasuryAccount = domainBuilder - .entity() - .customize(e -> e.type(EntityType.ACCOUNT).deleted(false).evmAddress(null)) - .persist(); + testWeb3jService.setSender(toAddress(sender.toEntityId()).toHexString()); + testWeb3jService.setValue(value); + final var token = populateHederaToken( contract.getContractAddress(), TokenTypeEnum.FUNGIBLE_COMMON, treasuryAccount.toEntityId()); @@ -729,13 +741,13 @@ void createFungibleTokenWithCustomFees() throws Exception { getAliasFromEntity(feeCollector)); // When - testWeb3jService.setValue(value); final var functionCall = contract.call_createFungibleTokenWithCustomFeesExternal( token, initialSupply, decimals, List.of(fixedFee), List.of(fractionalFee)); final var result = functionCall.send(); final var contractFunctionProvider = ContractFunctionProviderRecord.builder() .contractAddress(Address.fromHexString(contract.getContractAddress())) + .sender(toAddress(sender.toEntityId())) .value(value) .build(); @@ -750,9 +762,14 @@ void createFungibleTokenWithCustomFees() throws Exception { void createNonFungibleToken() throws Exception { // Given var value = 10000L * 100_000_000L; + final var sender = accountEntityPersist(); + + accountBalanceRecordsPersist(sender); final var contract = testWeb3jService.deploy(ModificationPrecompileTestContract::deploy); + testWeb3jService.setSender(toAddress(sender.toEntityId()).toHexString()); + final var treasuryAccount = domainBuilder .entity() .customize(e -> e.type(EntityType.ACCOUNT).deleted(false).evmAddress(null)) @@ -767,6 +784,7 @@ void createNonFungibleToken() throws Exception { final var contractFunctionProvider = ContractFunctionProviderRecord.builder() .contractAddress(Address.fromHexString(contract.getContractAddress())) + .sender(toAddress(sender.toEntityId())) .value(value) .build(); @@ -781,12 +799,20 @@ void createNonFungibleToken() throws Exception { void createNonFungibleTokenWithCustomFees() throws Exception { // Given var value = 10000L * 100_000_000L; + final var sender = accountEntityPersist(); + + accountBalanceRecordsPersist(sender); final var tokenForDenomination = persistFungibleToken(); final var feeCollector = accountEntityWithEvmAddressPersist(); + tokenAccountPersist(tokenForDenomination, feeCollector); + final var contract = testWeb3jService.deploy(ModificationPrecompileTestContract::deploy); + testWeb3jService.setSender(toAddress(sender.toEntityId()).toHexString()); + testWeb3jService.setValue(value); + final var treasuryAccount = domainBuilder .entity() .customize(e -> e.type(EntityType.ACCOUNT).deleted(false).evmAddress(null)) @@ -816,6 +842,7 @@ void createNonFungibleTokenWithCustomFees() throws Exception { final var contractFunctionProvider = ContractFunctionProviderRecord.builder() .contractAddress(Address.fromHexString(contract.getContractAddress())) + .sender(toAddress(sender.toEntityId())) .value(value) .build(); // Then @@ -1190,6 +1217,11 @@ void cryptoTransferHbars() throws Exception { final var receiver = accountEntityWithEvmAddressPersist(); final var payer = accountEntityWithEvmAddressPersist(); + long timestampForBalances = payer.getCreatedTimestamp(); + persistAccountBalance(payer, payer.getBalance()); + persistAccountBalance(sender, sender.getBalance(), timestampForBalances); + persistAccountBalance(treasuryEntity, treasuryEntity.getBalance(), timestampForBalances); + // When testWeb3jService.setSender(getAliasFromEntity(payer)); final var transferList = new TransferList(List.of( @@ -1214,10 +1246,19 @@ void cryptoTransferToken() throws Exception { final var receiver = accountEntityWithEvmAddressPersist(); final var payer = accountEntityWithEvmAddressPersist(); - tokenAccountPersist(tokenEntity, payer); + long timestampForBalances = payer.getCreatedTimestamp(); + persistAccountBalance(payer, payer.getBalance()); + persistTokenBalance(payer, tokenEntity, timestampForBalances); + + persistAccountBalance(sender, sender.getBalance(), timestampForBalances); + persistTokenBalance(sender, tokenEntity, timestampForBalances); + + persistTokenBalance(receiver, tokenEntity, timestampForBalances); + persistAccountBalance(treasuryEntity, treasuryEntity.getBalance(), timestampForBalances); + tokenAccountPersist(tokenEntity, sender); tokenAccountPersist(tokenEntity, receiver); - + tokenAccountPersist(tokenEntity, payer); // When testWeb3jService.setSender(getAliasFromEntity(payer)); final var tokenTransferList = new TokenTransferList( @@ -1246,9 +1287,20 @@ void cryptoTransferHbarsAndToken() throws Exception { final var receiver = accountEntityWithEvmAddressPersist(); final var payer = accountEntityWithEvmAddressPersist(); - tokenAccountPersist(tokenEntity, payer); + long timestampForBalances = payer.getCreatedTimestamp(); + persistAccountBalance(payer, payer.getBalance()); + persistTokenBalance(payer, tokenEntity, timestampForBalances); + + persistAccountBalance(sender, sender.getBalance(), timestampForBalances); + persistTokenBalance(sender, tokenEntity, timestampForBalances); + + persistTokenBalance(receiver, tokenEntity, timestampForBalances); + + persistAccountBalance(treasuryEntity, treasuryEntity.getBalance(), timestampForBalances); + tokenAccountPersist(tokenEntity, sender); tokenAccountPersist(tokenEntity, receiver); + tokenAccountPersist(tokenEntity, payer); // When testWeb3jService.setSender(getAliasFromEntity(payer)); @@ -1409,7 +1461,9 @@ private void verifyEthCallAndEstimateGas( private HederaToken populateHederaToken( final String contractAddress, final TokenTypeEnum tokenType, final EntityId treasuryAccountId) { - final var autoRenewAccount = accountEntityWithEvmAddressPersist(); + final var autoRenewAccount = + accountEntityWithEvmAddressPersist(); // the account that is going to be charged for token renewal upon + // expiration final var tokenEntity = domainBuilder .entity() .customize(e -> e.type(EntityType.TOKEN).autoRenewAccountId(autoRenewAccount.getId())) @@ -1419,21 +1473,26 @@ private HederaToken populateHederaToken( .customize(t -> t.tokenId(tokenEntity.getId()).type(tokenType).treasuryAccountId(treasuryAccountId)) .persist(); - final var supplyKey = - new KeyValue(Boolean.FALSE, contractAddress, new byte[0], new byte[0], Address.ZERO.toHexString()); + final var supplyKey = new KeyValue( + Boolean.FALSE, + contractAddress, + new byte[0], + new byte[0], + Address.ZERO.toHexString()); // the key needed for token minting or burning final var keys = new ArrayList(); keys.add(new TokenKey(AbstractContractCallServiceTest.KeyType.SUPPLY_KEY.getKeyTypeNumeric(), supplyKey)); + return new HederaToken( token.getName(), token.getSymbol(), - getAddressFromEntityId(treasuryAccountId), - tokenEntity.getMemo(), + getAddressFromEntityId(treasuryAccountId), // id of the account holding the initial token supply + tokenEntity.getMemo(), // token description encoded in UTF-8 format true, BigInteger.valueOf(10_000L), false, keys, new Expiry( - BigInteger.valueOf(Time.getCurrent().currentTimeMillis() + 1_000_000_000), + BigInteger.valueOf(Instant.now().getEpochSecond() + 8_000_000L), getAliasFromEntity(autoRenewAccount), BigInteger.valueOf(8_000_000))); } diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallServiceTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallServiceTest.java index ca874f9b1b4..10fa02960b9 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallServiceTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/ContractCallServiceTest.java @@ -63,7 +63,6 @@ import com.hedera.services.utils.EntityIdUtils; import io.github.bucket4j.Bucket; import jakarta.annotation.PostConstruct; -import jakarta.annotation.Resource; import java.lang.reflect.Method; import java.math.BigInteger; import java.util.Arrays; @@ -109,9 +108,6 @@ class ContractCallServiceTest extends AbstractContractCallServiceTest { @Autowired private TransactionExecutionService transactionExecutionService; - @Resource - private ContractExecutionService contractExecutionService; - private static Stream provideBlockTypes() { return Stream.of( BlockType.EARLIEST, @@ -147,6 +143,10 @@ private static String toHexWith64LeadingZeros(final Long value) { return result; } + private static Stream provideParametersForErcPrecompileExceptionalHalt() { + return Stream.of(Arguments.of(CallType.ETH_CALL, 1), Arguments.of(CallType.ETH_ESTIMATE_GAS, 2)); + } + @Override @BeforeEach protected void setup() { @@ -193,31 +193,35 @@ void pureCall() throws Exception { void pureCallModularizedServices() throws Exception { // Given final var modularizedServicesFlag = mirrorNodeEvmProperties.isModularizedServices(); - mirrorNodeEvmProperties.setModularizedServices(true); - Method postConstructMethod = Arrays.stream(MirrorNodeState.class.getDeclaredMethods()) - .filter(method -> method.isAnnotationPresent(PostConstruct.class)) - .findFirst() - .orElseThrow(() -> new IllegalStateException("@PostConstruct method not found")); + final var backupProperties = mirrorNodeEvmProperties.getProperties(); - postConstructMethod.setAccessible(true); // Make the method accessible - postConstructMethod.invoke(state); + try { + mirrorNodeEvmProperties.setModularizedServices(true); + Method postConstructMethod = Arrays.stream(MirrorNodeState.class.getDeclaredMethods()) + .filter(method -> method.isAnnotationPresent(PostConstruct.class)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("@PostConstruct method not found")); - final var backupProperties = mirrorNodeEvmProperties.getProperties(); - final Map propertiesMap = new HashMap<>(); - propertiesMap.put("contracts.maxRefundPercentOfGasLimit", "100"); - propertiesMap.put("contracts.maxGasPerSec", "15000000"); - mirrorNodeEvmProperties.setProperties(propertiesMap); + postConstructMethod.setAccessible(true); // Make the method accessible + postConstructMethod.invoke(state); - final var contract = testWeb3jService.deploy(EthCall::deploy); - meterRegistry.clear(); // Clear it as the contract deploy increases the gas limit metric + final Map propertiesMap = new HashMap<>(); + propertiesMap.put("contracts.maxRefundPercentOfGasLimit", "100"); + propertiesMap.put("contracts.maxGasPerSec", "15000000"); + mirrorNodeEvmProperties.setProperties(propertiesMap); - // When - contract.call_multiplySimpleNumbers().send(); + final var contract = testWeb3jService.deploy(EthCall::deploy); + meterRegistry.clear(); // Clear it as the contract deploy increases the gas limit metric - // Then - // Restore changed property values. - mirrorNodeEvmProperties.setModularizedServices(modularizedServicesFlag); - mirrorNodeEvmProperties.setProperties(backupProperties); + // When + contract.call_multiplySimpleNumbers().send(); + + // Then + // Restore changed property values. + } finally { + mirrorNodeEvmProperties.setModularizedServices(modularizedServicesFlag); + mirrorNodeEvmProperties.setProperties(backupProperties); + } } @ParameterizedTest @@ -972,10 +976,6 @@ private ContractExecutionParameters getContractExecutionParametersWithValue( .build(); } - private static Stream provideParametersForErcPrecompileExceptionalHalt() { - return Stream.of(Arguments.of(CallType.ETH_CALL, 1), Arguments.of(CallType.ETH_ESTIMATE_GAS, 2)); - } - private Entity accountPersist() { return domainBuilder.entity().persist(); } diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/InternalCallsTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/InternalCallsTest.java index 94e2065ce22..08a834a5336 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/InternalCallsTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/InternalCallsTest.java @@ -21,8 +21,11 @@ import static com.hedera.mirror.web3.service.model.CallServiceParameters.CallType.ETH_CALL; import static com.hedera.mirror.web3.utils.ContractCallTestUtil.TRANSACTION_GAS_LIMIT; import static com.hedera.mirror.web3.validation.HexValidator.HEX_PREFIX; +import static com.hederahashgraph.api.proto.java.ResponseCodeEnum.CONTRACT_REVERT_EXECUTED; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import com.hedera.mirror.web3.exception.MirrorEvmTransactionException; import com.hedera.mirror.web3.web3j.generated.InternalCaller; import org.apache.tuweni.bytes.Bytes; import org.junit.jupiter.api.Test; @@ -69,7 +72,12 @@ void sendToNonExistingAccount() throws Exception { meterRegistry.clear(); final var result = contract.call_sendTo(NON_EXISTING_ADDRESS).send(); - assertThat(result).isEqualTo(Boolean.TRUE); + if (!mirrorNodeEvmProperties.isModularizedServices()) { + assertThat(result).isEqualTo(Boolean.TRUE); + } else { + // In the mod code, there is a check if the address is an alias and in this case it is not. + assertThat(result).isEqualTo(Boolean.FALSE); + } assertGasLimit(TRANSACTION_GAS_LIMIT); } @@ -77,9 +85,16 @@ void sendToNonExistingAccount() throws Exception { void transferToNonExistingAccount() throws Exception { final var contract = testWeb3jService.deploy(InternalCaller::deploy); meterRegistry.clear(); - contract.send_transferTo(NON_EXISTING_ADDRESS).send(); - - assertThat(testWeb3jService.getTransactionResult()).isEqualTo(HEX_PREFIX); + final var functionCall = contract.send_transferTo(NON_EXISTING_ADDRESS); + if (!mirrorNodeEvmProperties.isModularizedServices()) { + functionCall.send(); + assertThat(testWeb3jService.getTransactionResult()).isEqualTo(HEX_PREFIX); + } else { + // In the mod code, there is a check if the address is an alias and in this case it is not. + assertThatThrownBy(functionCall::send) + .isInstanceOf(MirrorEvmTransactionException.class) + .hasMessage(CONTRACT_REVERT_EXECUTED.name()); + } assertGasLimit(TRANSACTION_GAS_LIMIT); } diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/TransactionExecutionServiceTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/TransactionExecutionServiceTest.java index cc60725530f..d527b23042d 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/TransactionExecutionServiceTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/service/TransactionExecutionServiceTest.java @@ -22,7 +22,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import com.hedera.hapi.node.base.ResponseCodeEnum; @@ -31,11 +30,11 @@ import com.hedera.hapi.node.transaction.TransactionReceipt; import com.hedera.hapi.node.transaction.TransactionRecord; import com.hedera.mirror.web3.common.ContractCallContext; +import com.hedera.mirror.web3.evm.contracts.execution.traceability.MirrorOperationTracer; import com.hedera.mirror.web3.evm.contracts.execution.traceability.OpcodeTracer; import com.hedera.mirror.web3.evm.contracts.execution.traceability.OpcodeTracerOptions; import com.hedera.mirror.web3.evm.properties.MirrorNodeEvmProperties; import com.hedera.mirror.web3.exception.MirrorEvmTransactionException; -import com.hedera.mirror.web3.service.TransactionExecutionService.ExecutorFactory; import com.hedera.mirror.web3.service.model.CallServiceParameters; import com.hedera.mirror.web3.service.model.CallServiceParameters.CallType; import com.hedera.mirror.web3.service.model.ContractExecutionParameters; @@ -65,7 +64,6 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; -import org.mockito.MockedStatic; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -76,16 +74,16 @@ class TransactionExecutionServiceTest { private State mirrorNodeState; @Mock - private MirrorNodeEvmProperties mirrorNodeEvmProperties; + private OpcodeTracer opcodeTracer; @Mock - private OpcodeTracer opcodeTracer; + private MirrorOperationTracer mirrorOperationTracer; @Mock private TransactionExecutor transactionExecutor; @Mock - private ContractCallContext contractCallContext; + private TransactionExecutorFactory transactionExecutorFactory; @Mock private MeterProvider gasUsedCounter; @@ -100,8 +98,13 @@ private static Stream provideCallData() { @BeforeEach void setUp() { - transactionExecutionService = - new TransactionExecutionService(mirrorNodeState, mirrorNodeEvmProperties, opcodeTracer); + transactionExecutionService = new TransactionExecutionService( + mirrorNodeState, + new MirrorNodeEvmProperties(), + opcodeTracer, + mirrorOperationTracer, + transactionExecutorFactory); + when(transactionExecutorFactory.get()).thenReturn(transactionExecutor); } @ParameterizedTest @@ -113,29 +116,20 @@ void setUp() { }) void testExecuteContractCallSuccess(String senderAddressHex) { // Given - try (MockedStatic executorFactoryMock = mockStatic(ExecutorFactory.class); - MockedStatic contractCallContextMock = mockStatic(ContractCallContext.class)) { - - // Set up mock behaviors for ExecutorFactory - executorFactoryMock - .when(() -> ExecutorFactory.newExecutor(any(), any(), any())) - .thenReturn(transactionExecutor); - - // Set up mock behaviors for ContractCallContext - contractCallContextMock.when(ContractCallContext::get).thenReturn(contractCallContext); - when(contractCallContext.getOpcodeTracerOptions()).thenReturn(new OpcodeTracerOptions()); + ContractCallContext.run(context -> { + context.setOpcodeTracerOptions(new OpcodeTracerOptions()); // Mock the SingleTransactionRecord and TransactionRecord - SingleTransactionRecord singleTransactionRecord = mock(SingleTransactionRecord.class); - TransactionRecord transactionRecord = mock(TransactionRecord.class); - TransactionReceipt transactionReceipt = mock(TransactionReceipt.class); + var singleTransactionRecord = mock(SingleTransactionRecord.class); + var transactionRecord = mock(TransactionRecord.class); + var transactionReceipt = mock(TransactionReceipt.class); // Simulate SUCCESS status in the receipt when(transactionReceipt.status()).thenReturn(ResponseCodeEnum.SUCCESS); when(transactionRecord.receiptOrThrow()).thenReturn(transactionReceipt); when(singleTransactionRecord.transactionRecord()).thenReturn(transactionRecord); - ContractFunctionResult contractFunctionResult = mock(ContractFunctionResult.class); + var contractFunctionResult = mock(ContractFunctionResult.class); when(contractFunctionResult.gasUsed()).thenReturn(DEFAULT_GAS); when(contractFunctionResult.contractCallResult()).thenReturn(Bytes.EMPTY); @@ -156,8 +150,7 @@ void testExecuteContractCallSuccess(String senderAddressHex) { any(TransactionBody.class), any(Instant.class), any(OperationTracer[].class))) .thenReturn(List.of(singleTransactionRecord)); - CallServiceParameters callServiceParameters = - buildServiceParams(false, org.apache.tuweni.bytes.Bytes.EMPTY, senderAddress); + var callServiceParameters = buildServiceParams(false, org.apache.tuweni.bytes.Bytes.EMPTY, senderAddress); // When var result = transactionExecutionService.execute(callServiceParameters, DEFAULT_GAS, gasUsedCounter); @@ -166,7 +159,8 @@ void testExecuteContractCallSuccess(String senderAddressHex) { assertThat(result).isNotNull(); assertThat(result.getGasUsed()).isEqualTo(DEFAULT_GAS); assertThat(result.getRevertReason()).isNotPresent(); - } + return null; + }); } @ParameterizedTest @@ -175,30 +169,21 @@ void testExecuteContractCallSuccess(String senderAddressHex) { "INVALID_TOKEN_ID,CONTRACT_REVERT_EXECUTED,''", "0x,INVALID_TOKEN_ID,''" }) + @SuppressWarnings("unused") void testExecuteContractCallFailureWithErrorMessage( final String errorMessage, final ResponseCodeEnum responseCode, final String detail) { // Given - try (MockedStatic executorFactoryMock = mockStatic(ExecutorFactory.class); - MockedStatic contractCallContextMock = mockStatic(ContractCallContext.class)) { - - // Set up mock behaviors for ExecutorFactory - executorFactoryMock - .when(() -> ExecutorFactory.newExecutor(any(), any(), any())) - .thenReturn(transactionExecutor); - - // Set up mock behaviors for ContractCallContext - contractCallContextMock.when(ContractCallContext::get).thenReturn(contractCallContext); - + ContractCallContext.run(context -> { // Mock the SingleTransactionRecord and TransactionRecord - SingleTransactionRecord singleTransactionRecord = mock(SingleTransactionRecord.class); - TransactionRecord transactionRecord = mock(TransactionRecord.class); - TransactionReceipt transactionReceipt = mock(TransactionReceipt.class); + var singleTransactionRecord = mock(SingleTransactionRecord.class); + var transactionRecord = mock(TransactionRecord.class); + var transactionReceipt = mock(TransactionReceipt.class); when(transactionReceipt.status()).thenReturn(responseCode); when(transactionRecord.receiptOrThrow()).thenReturn(transactionReceipt); when(singleTransactionRecord.transactionRecord()).thenReturn(transactionRecord); - ContractFunctionResult contractFunctionResult = mock(ContractFunctionResult.class); + var contractFunctionResult = mock(ContractFunctionResult.class); when(transactionRecord.contractCallResult()).thenReturn(contractFunctionResult); when(contractFunctionResult.errorMessage()).thenReturn(errorMessage); when(gasUsedCounter.withTags(anyString(), anyString(), anyString(), anyString())) @@ -209,8 +194,7 @@ void testExecuteContractCallFailureWithErrorMessage( any(TransactionBody.class), any(Instant.class), any(OperationTracer[].class))) .thenReturn(List.of(singleTransactionRecord)); - CallServiceParameters callServiceParameters = - buildServiceParams(false, org.apache.tuweni.bytes.Bytes.EMPTY, Address.ZERO); + var callServiceParameters = buildServiceParams(false, org.apache.tuweni.bytes.Bytes.EMPTY, Address.ZERO); // Then assertThatThrownBy(() -> @@ -218,27 +202,19 @@ void testExecuteContractCallFailureWithErrorMessage( .isInstanceOf(MirrorEvmTransactionException.class) .hasMessageContaining(responseCode.name()) .hasFieldOrPropertyWithValue("detail", detail); - } + return null; + }); } + @SuppressWarnings("unused") @Test void testExecuteContractCallFailureOnPreChecks() { // Given - try (MockedStatic executorFactoryMock = mockStatic(ExecutorFactory.class); - MockedStatic contractCallContextMock = mockStatic(ContractCallContext.class)) { - - // Set up mock behaviors for ExecutorFactory - executorFactoryMock - .when(() -> ExecutorFactory.newExecutor(any(), any(), any())) - .thenReturn(transactionExecutor); - - // Set up mock behaviors for ContractCallContext - contractCallContextMock.when(ContractCallContext::get).thenReturn(contractCallContext); - + ContractCallContext.run(context -> { // Mock the SingleTransactionRecord and TransactionRecord - SingleTransactionRecord singleTransactionRecord = mock(SingleTransactionRecord.class); - TransactionRecord transactionRecord = mock(TransactionRecord.class); - TransactionReceipt transactionReceipt = mock(TransactionReceipt.class); + var singleTransactionRecord = mock(SingleTransactionRecord.class); + var transactionRecord = mock(TransactionRecord.class); + var transactionReceipt = mock(TransactionReceipt.class); when(transactionRecord.receiptOrThrow()).thenReturn(transactionReceipt); when(transactionReceipt.status()).thenReturn(ResponseCodeEnum.INVALID_ACCOUNT_ID); @@ -249,15 +225,15 @@ void testExecuteContractCallFailureOnPreChecks() { any(TransactionBody.class), any(Instant.class), any(OperationTracer[].class))) .thenReturn(List.of(singleTransactionRecord)); - CallServiceParameters callServiceParameters = - buildServiceParams(false, org.apache.tuweni.bytes.Bytes.EMPTY, Address.ZERO); + var callServiceParameters = buildServiceParams(false, org.apache.tuweni.bytes.Bytes.EMPTY, Address.ZERO); // Then assertThatThrownBy(() -> transactionExecutionService.execute(callServiceParameters, DEFAULT_GAS, gasUsedCounter)) .isInstanceOf(MirrorEvmTransactionException.class) .hasMessageContaining(ResponseCodeEnum.INVALID_ACCOUNT_ID.name()); - } + return null; + }); } // NestedCalls.BINARY @@ -265,28 +241,19 @@ void testExecuteContractCallFailureOnPreChecks() { @MethodSource("provideCallData") void testExecuteContractCreateSuccess(org.apache.tuweni.bytes.Bytes callData) { // Given - try (MockedStatic executorFactoryMock = mockStatic(ExecutorFactory.class); - MockedStatic contractCallContextMock = mockStatic(ContractCallContext.class)) { - - // Set up mock behaviors for ExecutorFactory - executorFactoryMock - .when(() -> ExecutorFactory.newExecutor(any(), any(), any())) - .thenReturn(transactionExecutor); - - // Set up mock behaviors for ContractCallContext - contractCallContextMock.when(ContractCallContext::get).thenReturn(contractCallContext); - when(contractCallContext.getOpcodeTracerOptions()).thenReturn(new OpcodeTracerOptions()); + ContractCallContext.run(context -> { + context.setOpcodeTracerOptions(new OpcodeTracerOptions()); // Mock the SingleTransactionRecord and TransactionRecord - SingleTransactionRecord singleTransactionRecord = mock(SingleTransactionRecord.class); - TransactionRecord transactionRecord = mock(TransactionRecord.class); - TransactionReceipt transactionReceipt = mock(TransactionReceipt.class); + var singleTransactionRecord = mock(SingleTransactionRecord.class); + var transactionRecord = mock(TransactionRecord.class); + var transactionReceipt = mock(TransactionReceipt.class); when(transactionReceipt.status()).thenReturn(ResponseCodeEnum.SUCCESS); when(transactionRecord.receiptOrThrow()).thenReturn(transactionReceipt); when(singleTransactionRecord.transactionRecord()).thenReturn(transactionRecord); - ContractFunctionResult contractFunctionResult = mock(ContractFunctionResult.class); + var contractFunctionResult = mock(ContractFunctionResult.class); when(contractFunctionResult.gasUsed()).thenReturn(DEFAULT_GAS); when(contractFunctionResult.contractCallResult()).thenReturn(Bytes.EMPTY); @@ -298,7 +265,7 @@ void testExecuteContractCreateSuccess(org.apache.tuweni.bytes.Bytes callData) { any(TransactionBody.class), any(Instant.class), any(OperationTracer[].class))) .thenReturn(List.of(singleTransactionRecord)); - CallServiceParameters callServiceParameters = buildServiceParams(true, callData, Address.ZERO); + var callServiceParameters = buildServiceParams(true, callData, Address.ZERO); // When var result = transactionExecutionService.execute(callServiceParameters, DEFAULT_GAS, gasUsedCounter); @@ -307,7 +274,8 @@ void testExecuteContractCreateSuccess(org.apache.tuweni.bytes.Bytes callData) { assertThat(result).isNotNull(); assertThat(result.getGasUsed()).isEqualTo(DEFAULT_GAS); assertThat(result.getRevertReason()).isNotPresent(); - } + return null; + }); } private CallServiceParameters buildServiceParams( diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/MirrorNodeStateIntegrationTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/MirrorNodeStateIntegrationTest.java index ee847660722..0572e00b225 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/MirrorNodeStateIntegrationTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/MirrorNodeStateIntegrationTest.java @@ -20,6 +20,9 @@ import com.hedera.mirror.web3.Web3IntegrationTest; import com.hedera.mirror.web3.evm.properties.MirrorNodeEvmProperties; +import com.hedera.mirror.web3.state.singleton.BlockInfoSingleton; +import com.hedera.mirror.web3.state.singleton.DefaultSingleton; +import com.hedera.mirror.web3.state.singleton.RunningHashesSingleton; import com.hedera.node.app.fees.FeeService; import com.hedera.node.app.ids.EntityIdService; import com.hedera.node.app.records.BlockRecordService; @@ -88,8 +91,8 @@ void verifyServicesHaveAssignedDataSources() { // BlockRecordService Map> blockRecordServiceDataSources = Map.of( - "BLOCKS", AtomicReference.class, - "RUNNING_HASHES", AtomicReference.class); + "BLOCKS", BlockInfoSingleton.class, + "RUNNING_HASHES", RunningHashesSingleton.class); verifyServiceDataSources(states, BlockRecordService.NAME, blockRecordServiceDataSources); // FileService @@ -98,12 +101,12 @@ void verifyServicesHaveAssignedDataSources() { // CongestionThrottleService Map> congestionThrottleServiceDataSources = Map.of( - "THROTTLE_USAGE_SNAPSHOTS", AtomicReference.class, - "CONGESTION_LEVEL_STARTS", AtomicReference.class); + "THROTTLE_USAGE_SNAPSHOTS", DefaultSingleton.class, + "CONGESTION_LEVEL_STARTS", DefaultSingleton.class); verifyServiceDataSources(states, CongestionThrottleService.NAME, congestionThrottleServiceDataSources); // FeeService - Map> feeServiceDataSources = Map.of("MIDNIGHT_RATES", AtomicReference.class); + Map> feeServiceDataSources = Map.of("MIDNIGHT_RATES", DefaultSingleton.class); verifyServiceDataSources(states, FeeService.NAME, feeServiceDataSources); // ContractService @@ -117,7 +120,7 @@ void verifyServicesHaveAssignedDataSources() { verifyServiceDataSources(states, RecordCacheService.NAME, recordCacheServiceDataSources); // EntityIdService - Map> entityIdServiceDataSources = Map.of("ENTITY_ID", AtomicReference.class); + Map> entityIdServiceDataSources = Map.of("ENTITY_ID", DefaultSingleton.class); verifyServiceDataSources(states, EntityIdService.NAME, entityIdServiceDataSources); // TokenService diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/MirrorNodeStateTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/MirrorNodeStateTest.java index 3c25c443891..303e1907bf7 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/MirrorNodeStateTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/MirrorNodeStateTest.java @@ -26,6 +26,7 @@ import com.hedera.mirror.web3.state.core.MapReadableStates; import com.hedera.mirror.web3.state.core.MapWritableKVState; import com.hedera.mirror.web3.state.core.MapWritableStates; +import com.hedera.mirror.web3.state.singleton.DefaultSingleton; import com.hedera.node.app.ids.EntityIdService; import com.hedera.node.app.service.contract.ContractService; import com.hedera.node.app.service.file.FileService; @@ -45,7 +46,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -151,9 +151,9 @@ void testRemoveService() { final var testStates = new HashMap<>(Map.of( ContractBytecodeReadableKVState.KEY, - Map.of(ContractBytecodeReadableKVState.KEY, contractBytecodeReadableKVState), + Map.of(ContractBytecodeReadableKVState.KEY, contractBytecodeReadableKVState), ContractStorageReadableKVState.KEY, - Map.of(ContractStorageReadableKVState.KEY, contractStorageReadableKVState))); + Map.of(ContractStorageReadableKVState.KEY, contractStorageReadableKVState))); final var newState = mirrorNodeState.addService("NEW", testStates); assertThat(newState.getReadableStates("NEW").contains(ContractBytecodeReadableKVState.KEY)) .isTrue(); @@ -234,10 +234,13 @@ void testGetReadableStateForUnsupportedService() { @Test void testGetReadableStatesWithSingleton() { final var stateWithSingleton = buildStateObject(); - stateWithSingleton.addService(EntityIdService.NAME, Map.of("EntityId", new AtomicReference<>(1L))); + final var key = "EntityId"; + final var singleton = new DefaultSingleton(key); + singleton.set(1L); + stateWithSingleton.addService(EntityIdService.NAME, Map.of(key, singleton)); final var readableStates = stateWithSingleton.getReadableStates(EntityIdService.NAME); - assertThat(readableStates.contains("EntityId")).isTrue(); - assertThat(readableStates.getSingleton("EntityId").get()).isEqualTo(1L); + assertThat(readableStates.contains(key)).isTrue(); + assertThat(readableStates.getSingleton(key).get()).isEqualTo(1L); } @Test @@ -355,17 +358,22 @@ void testGetWritableStateForUnsupportedService() { @Test void testGetWritableStatesWithSingleton() { final var stateWithSingleton = buildStateObject(); - stateWithSingleton.addService(EntityIdService.NAME, Map.of("EntityId", new AtomicReference<>(1L))); + final var key = "EntityId"; + final var singleton = new DefaultSingleton(key); + singleton.set(1L); + stateWithSingleton.addService(EntityIdService.NAME, Map.of(key, singleton)); final var writableStates = stateWithSingleton.getWritableStates(EntityIdService.NAME); - assertThat(writableStates.contains("EntityId")).isTrue(); - assertThat(writableStates.getSingleton("EntityId").get()).isEqualTo(1L); + assertThat(writableStates.contains(key)).isTrue(); + assertThat(writableStates.getSingleton(key).get()).isEqualTo(1L); } @Test void testGetWritableStatesWithSingletonWithListeners() { final var stateWithSingleton = buildStateObject(); - final var ref = new AtomicReference<>(1L); - stateWithSingleton.addService(EntityIdService.NAME, Map.of("EntityId", ref)); + final var key = "EntityId"; + final var singleton = new DefaultSingleton(key); + singleton.set(1L); + stateWithSingleton.addService(EntityIdService.NAME, Map.of(key, singleton)); when(listener.stateTypes()).thenReturn(Set.of(StateType.SINGLETON)); stateWithSingleton.registerCommitListener(listener); @@ -460,17 +468,22 @@ private MirrorNodeState initStateAfterMigration() { new HashMap<>(Map.of(FileReadableKVState.KEY, Map.of(FileReadableKVState.KEY, fileReadableKVState))); final Map contractStateData = new HashMap<>(Map.of( ContractBytecodeReadableKVState.KEY, - Map.of(ContractBytecodeReadableKVState.KEY, contractBytecodeReadableKVState), + Map.of(ContractBytecodeReadableKVState.KEY, contractBytecodeReadableKVState), ContractStorageReadableKVState.KEY, - Map.of(ContractStorageReadableKVState.KEY, contractStorageReadableKVState))); + Map.of(ContractStorageReadableKVState.KEY, contractStorageReadableKVState))); final Map tokenStateData = new HashMap<>(Map.of( - AccountReadableKVState.KEY, Map.of(AccountReadableKVState.KEY, accountReadableKVState), - AirdropsReadableKVState.KEY, Map.of(AirdropsReadableKVState.KEY, airdropsReadableKVState), - AliasesReadableKVState.KEY, Map.of(AliasesReadableKVState.KEY, aliasesReadableKVState), - NftReadableKVState.KEY, Map.of(NftReadableKVState.KEY, nftReadableKVState), - TokenReadableKVState.KEY, Map.of(TokenReadableKVState.KEY, tokenReadableKVState), + AccountReadableKVState.KEY, + Map.of(AccountReadableKVState.KEY, accountReadableKVState), + AirdropsReadableKVState.KEY, + Map.of(AirdropsReadableKVState.KEY, airdropsReadableKVState), + AliasesReadableKVState.KEY, + Map.of(AliasesReadableKVState.KEY, aliasesReadableKVState), + NftReadableKVState.KEY, + Map.of(NftReadableKVState.KEY, nftReadableKVState), + TokenReadableKVState.KEY, + Map.of(TokenReadableKVState.KEY, tokenReadableKVState), TokenRelationshipReadableKVState.KEY, - Map.of(TokenRelationshipReadableKVState.KEY, tokenRelationshipReadableKVState))); + Map.of(TokenRelationshipReadableKVState.KEY, tokenRelationshipReadableKVState))); // Add service using the mock data source return buildStateObject() diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/components/SchemaRegistryImplTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/components/SchemaRegistryImplTest.java index ecb03849472..64165e5027e 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/components/SchemaRegistryImplTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/components/SchemaRegistryImplTest.java @@ -41,6 +41,7 @@ import com.swirlds.state.spi.ReadableStates; import java.util.EnumSet; import java.util.HashMap; +import java.util.List; import java.util.Set; import java.util.SortedSet; import java.util.concurrent.atomic.AtomicLong; @@ -80,15 +81,13 @@ class SchemaRegistryImplTest { @Mock private Codec mockCodec; - private Configuration appConfig; - private Configuration platformConfig; + private Configuration config; private SchemaRegistryImpl schemaRegistry; @BeforeEach void initialize() { - schemaRegistry = new SchemaRegistryImpl(schemaApplications); - appConfig = new ConfigProviderImpl().getConfiguration(); - platformConfig = new ConfigProviderImpl().getConfiguration(); + schemaRegistry = new SchemaRegistryImpl(List.of(), schemaApplications); + config = new ConfigProviderImpl().getConfiguration(); } @Test @@ -118,8 +117,8 @@ void testMigrateWithSingleSchema() { mirrorNodeState, previousVersion, networkInfo, - appConfig, - platformConfig, + config, + config, new HashMap<>(), new AtomicLong(), startupNetworks); @@ -140,8 +139,8 @@ void testMigrateWithMigrations() { mirrorNodeState, previousVersion, networkInfo, - appConfig, - platformConfig, + config, + config, new HashMap<>(), new AtomicLong(), startupNetworks); @@ -165,7 +164,7 @@ void testMigrateWithStateDefinitions() { StateDefinition stateDefinition = new StateDefinition("STATE", mockCodec, mockCodec, 123, true, false, false); - when(schema.statesToCreate(appConfig)) + when(schema.statesToCreate(config)) .thenReturn(Set.of(stateDefinitionSingleton, stateDefinitionQueue, stateDefinition)); schemaRegistry.register(schema); @@ -174,8 +173,8 @@ void testMigrateWithStateDefinitions() { mirrorNodeState, previousVersion, networkInfo, - appConfig, - platformConfig, + config, + config, new HashMap<>(), new AtomicLong(), startupNetworks); @@ -199,8 +198,8 @@ void testMigrateWithRestartApplication() { mirrorNodeState, previousVersion, networkInfo, - appConfig, - platformConfig, + config, + config, new HashMap<>(), new AtomicLong(), startupNetworks); @@ -215,8 +214,8 @@ void testNewMigrationContext() { previousVersion, readableStates, writableStates, - appConfig, - platformConfig, + config, + config, networkInfo, new AtomicLong(1), EMPTY_MAP, @@ -229,8 +228,8 @@ void testNewMigrationContext() { assertThat(c.previousVersion()).isEqualTo(previousVersion); assertThat(c.previousStates()).isEqualTo(readableStates); assertThat(c.newStates()).isEqualTo(writableStates); - assertThat(c.appConfig()).isEqualTo(appConfig); - assertThat(c.platformConfig()).isEqualTo(platformConfig); + assertThat(c.appConfig()).isEqualTo(config); + assertThat(c.platformConfig()).isEqualTo(config); assertThat(c.genesisNetworkInfo()).isEqualTo(networkInfo); assertThat(c.newEntityNum()).isEqualTo(1); assertThat(c.sharedValues()).isEqualTo(EMPTY_MAP); diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/components/ServicesRegistryImplTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/components/ServicesRegistryImplTest.java index b4a8a243bfb..72f71d3758b 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/components/ServicesRegistryImplTest.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/components/ServicesRegistryImplTest.java @@ -21,6 +21,7 @@ import com.hedera.node.app.ids.EntityIdService; import com.hedera.node.app.service.file.impl.FileServiceImpl; import com.swirlds.state.lifecycle.Service; +import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -30,7 +31,7 @@ class ServicesRegistryImplTest { @BeforeEach void setUp() { - servicesRegistry = new ServicesRegistryImpl(); + servicesRegistry = new ServicesRegistryImpl(List.of()); } @Test diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/singleton/BlockInfoSingletonTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/singleton/BlockInfoSingletonTest.java new file mode 100644 index 00000000000..ea23ca203a5 --- /dev/null +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/singleton/BlockInfoSingletonTest.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.web3.state.singleton; + +import static com.hedera.mirror.web3.state.Utils.convertToTimestamp; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.when; + +import com.hedera.hapi.node.state.blockrecords.BlockInfo; +import com.hedera.mirror.common.domain.DomainBuilder; +import com.hedera.mirror.common.domain.transaction.RecordFile; +import com.hedera.mirror.web3.common.ContractCallContext; +import com.hedera.mirror.web3.evm.properties.MirrorNodeEvmProperties; +import com.hedera.mirror.web3.repository.RecordFileRepository; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class BlockInfoSingletonTest { + + private final DomainBuilder domainBuilder = new DomainBuilder(); + private final MirrorNodeEvmProperties properties = new MirrorNodeEvmProperties(); + + private BlockInfoSingleton blockInfoSingleton; + + @Mock + private RecordFileRepository recordFileRepository; + + @BeforeEach + void setup() { + properties.setProperties(Map.of("hedera.recordStream.numOfBlockHashesInState", "2")); + blockInfoSingleton = new BlockInfoSingleton(properties, recordFileRepository); + } + + @ValueSource(longs = {0, 1, 2}) + @ParameterizedTest + void get(long startIndex) { + ContractCallContext.run(context -> { + var recordFile1 = recordFile(startIndex); + var recordFile2 = recordFile(startIndex + 1L); + var recordFile3 = recordFile(startIndex + 2L); + var recordFiles = new ArrayList<>(List.of(recordFile1, recordFile2)); + + when(recordFileRepository.findByIndexRange(startIndex, startIndex + 1)) + .thenReturn(recordFiles); + context.setRecordFile(recordFile3); + + assertThat(blockInfoSingleton.get()) + .isEqualTo(BlockInfo.newBuilder() + .blockHashes(Bytes.fromHex( + recordFile1.getHash() + recordFile2.getHash() + recordFile3.getHash())) + .consTimeOfLastHandledTxn(convertToTimestamp(recordFile3.getConsensusEnd())) + .firstConsTimeOfCurrentBlock(convertToTimestamp(recordFile3.getConsensusEnd())) + .firstConsTimeOfLastBlock(convertToTimestamp(recordFile3.getConsensusStart())) + .lastBlockNumber(recordFile3.getIndex()) + .migrationRecordsStreamed(true) + .build()); + return null; + }); + } + + @Test + void getGenesis() { + ContractCallContext.run(context -> { + var recordFile = recordFile(0L); + context.setRecordFile(recordFile); + + assertThat(blockInfoSingleton.get()) + .isEqualTo(BlockInfo.newBuilder() + .blockHashes(Bytes.fromHex(recordFile.getHash())) + .consTimeOfLastHandledTxn(convertToTimestamp(recordFile.getConsensusEnd())) + .firstConsTimeOfCurrentBlock(convertToTimestamp(recordFile.getConsensusEnd())) + .firstConsTimeOfLastBlock(convertToTimestamp(recordFile.getConsensusStart())) + .lastBlockNumber(recordFile.getIndex()) + .migrationRecordsStreamed(true) + .build()); + return null; + }); + } + + @Test + void key() { + assertThat(blockInfoSingleton.getKey()).isEqualTo("BLOCKS"); + } + + @Test + void set() { + ContractCallContext.run(context -> { + var recordFile = recordFile(0L); + context.setRecordFile(recordFile); + blockInfoSingleton.set(BlockInfo.DEFAULT); + assertThat(blockInfoSingleton.get()).isNotEqualTo(BlockInfo.DEFAULT); + return null; + }); + } + + private RecordFile recordFile(long index) { + return domainBuilder.recordFile().customize(r -> r.index(index)).get(); + } +} diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/singleton/RunningHashesSingletonTest.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/singleton/RunningHashesSingletonTest.java new file mode 100644 index 00000000000..99d51beb923 --- /dev/null +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/state/singleton/RunningHashesSingletonTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.web3.state.singleton; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.hedera.hapi.node.state.blockrecords.RunningHashes; +import com.hedera.mirror.common.domain.DomainBuilder; +import com.hedera.mirror.web3.common.ContractCallContext; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import org.junit.jupiter.api.Test; + +class RunningHashesSingletonTest { + + private final DomainBuilder domainBuilder = new DomainBuilder(); + private final RunningHashesSingleton runningHashesSingleton = new RunningHashesSingleton(); + + @Test + void get() { + ContractCallContext.run(context -> { + var recordFile = domainBuilder.recordFile().get(); + context.setRecordFile(recordFile); + assertThat(runningHashesSingleton.get()) + .isEqualTo(RunningHashes.newBuilder() + .runningHash(Bytes.fromHex(recordFile.getHash())) + .build()); + return null; + }); + } + + @Test + void key() { + assertThat(runningHashesSingleton.getKey()).isEqualTo("RUNNING_HASHES"); + } + + @Test + void set() { + ContractCallContext.run(context -> { + var recordFile = domainBuilder.recordFile().get(); + context.setRecordFile(recordFile); + runningHashesSingleton.set(RunningHashes.DEFAULT); + assertThat(runningHashesSingleton.get()).isNotEqualTo(RunningHashes.DEFAULT); + return null; + }); + } +} diff --git a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/web3j/TestWeb3jService.java b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/web3j/TestWeb3jService.java index 2ebd200b35f..cb36f109e40 100644 --- a/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/web3j/TestWeb3jService.java +++ b/hedera-mirror-web3/src/test/java/com/hedera/mirror/web3/web3j/TestWeb3jService.java @@ -87,7 +87,7 @@ public class TestWeb3jService implements Web3jService { private boolean isEstimateGas = false; private String transactionResult; private Supplier estimatedGas; - private long value = 0L; + private long value = 0L; // the amount sent to the smart contract, if the contract function is payable. private boolean persistContract = true; private byte[] contractRuntime; private BlockType blockType = BlockType.LATEST;