Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update Snapshot Dirs #113

Merged
merged 2 commits into from
Jan 3, 2024
Merged

Update Snapshot Dirs #113

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6f04470
update snapshot dirs
jedleggett Jan 3, 2024
145648a
Merge branch 'master' into jl/update_snapshot_dirs
jedleggett Jan 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 35 additions & 29 deletions mev-programs/scripts/autosnapshot_inner.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,13 @@ check_env_vars_set() {
exit 1
fi

if [ -z "$SNAPSHOT_DIR" ]; then
echo "SNAPSHOT_DIR must be set"
if [ -z "$SNAPSHOT_OUTPUT_DIR" ]; then
echo "SNAPSHOT_OUTPUT_DIR must be set"
exit 1
fi

if [ -z "$SNAPSHOT_ARCHIVE_DIR" ]; then
echo "SNAPSHOT_ARCHIVE_DIR must be set"
exit 1
fi

Expand Down Expand Up @@ -107,15 +112,16 @@ get_snapshot_filename() {
# creates a snapshot for the given slot
create_snapshot_for_slot() {
local snapshot_slot=$1
local snapshot_dir=$2
local ledger_location=$3
local snapshot_output_dir=$2
local snapshot_archive_dir=$3
local ledger_location=$4

local snapshot_file
local exit_status

# produces snapshot in $snapshot_dir
# produces snapshot in $snapshot_output_dir
# shellcheck disable=SC2012
RUST_LOG=info solana-ledger-tool -l "$ledger_location" create-snapshot "$snapshot_slot" "$snapshot_dir"
RUST_LOG=info solana-ledger-tool -l "$ledger_location" create-snapshot --snapshot-archive-path "$snapshot_archive_dir" "$snapshot_slot" "$snapshot_output_dir"

# TODO: figure this out
# for some reason solana-ledger-tool error doesn't cause this script to exit out, so check status here
Expand Down Expand Up @@ -157,24 +163,24 @@ upload_file_to_gcloud() {

generate_stake_meta() {
local slot=$1
local snapshot_dir=$2
local snapshot_output_dir=$2
local stake_meta_filename=$3
local tip_distribution_program_id=$4
local tip_payment_program_id=$5

rm -rf "$snapshot_dir"/stake-meta.accounts
rm -rf "$snapshot_dir"/tmp*
rm -rf "$snapshot_output_dir"/stake-meta.accounts
rm -rf "$snapshot_output_dir"/tmp*
rm -rf /tmp/.tmp* # calculate hash stuff stored here

RUST_LOG=info solana-stake-meta-generator \
--ledger-path "$snapshot_dir" \
--ledger-path "$snapshot_output_dir" \
--tip-distribution-program-id "$tip_distribution_program_id" \
--out-path "$snapshot_dir/$stake_meta_filename" \
--out-path "$snapshot_output_dir/$stake_meta_filename" \
--snapshot-slot "$slot" \
--tip-payment-program-id "$tip_payment_program_id"

rm -rf "$snapshot_dir"/stake-meta.accounts
rm -rf "$snapshot_dir"/tmp*
rm -rf "$snapshot_output_dir"/stake-meta.accounts
rm -rf "$snapshot_output_dir"/tmp*
rm -rf /tmp/.tmp* # calculate hash stuff stored here
}

Expand Down Expand Up @@ -223,11 +229,11 @@ prune_old_snapshots() {
local to_delete_snapshot

# sorts by timestamp in filename
to_delete_stake=$(find "$SNAPSHOT_DIR" -type f -name 'stake-meta-[0-9]*.json' | sort | head -n -$NUM_SNAPSHOTS_TO_KEEP)
to_delete_merkle=$(find "$SNAPSHOT_DIR" -type f -name 'merkle-tree-[0-9]*.json' | sort | head -n -$NUM_SNAPSHOTS_TO_KEEP)
to_delete_snapshot=$(find "$SNAPSHOT_DIR" -type f -name 'snapshot-[0-9]*-[[:alnum:]]*.tar.zst' | sort | head -n -$NUM_SNAPSHOTS_TO_KEEP)
to_delete_stake=$(find "$SNAPSHOT_OUTPUT_DIR" -type f -name 'stake-meta-[0-9]*.json' | sort | head -n -$NUM_SNAPSHOTS_TO_KEEP)
to_delete_merkle=$(find "$SNAPSHOT_OUTPUT_DIR" -type f -name 'merkle-tree-[0-9]*.json' | sort | head -n -$NUM_SNAPSHOTS_TO_KEEP)
to_delete_snapshot=$(find "$SNAPSHOT_OUTPUT_DIR" -type f -name 'snapshot-[0-9]*-[[:alnum:]]*.tar.zst' | sort | head -n -$NUM_SNAPSHOTS_TO_KEEP)

echo "pruning $(echo "$to_delete_snapshot" | wc -w) snapshots in $SNAPSHOT_DIR"
echo "pruning $(echo "$to_delete_snapshot" | wc -w) snapshots in $SNAPSHOT_OUTPUT_DIR"
# shellcheck disable=SC2086
rm -f -v $to_delete_stake
# shellcheck disable=SC2086
Expand Down Expand Up @@ -269,8 +275,8 @@ main() {
check_env_vars_set

# make sure snapshot location exists and has a genesis file in it
mkdir -p "$SNAPSHOT_DIR"
cp "$LEDGER_DIR"/genesis.bin "$SNAPSHOT_DIR"
mkdir -p "$SNAPSHOT_OUTPUT_DIR"
jedleggett marked this conversation as resolved.
Show resolved Hide resolved
cp "$LEDGER_DIR"/genesis.bin "$SNAPSHOT_OUTPUT_DIR"

# ---------------------------------------------------------------------------
# Read epoch info off RPC and calculate previous epoch + previous epoch's last slot
Expand All @@ -290,7 +296,7 @@ main() {
echo "epoch_info: $epoch_info"
echo "previous_epoch_final_slot: $previous_epoch_final_slot"

FILE="$SNAPSHOT_DIR/$last_epoch.done"
FILE="$SNAPSHOT_OUTPUT_DIR/$last_epoch.done"
if [ -f "$FILE" ]; then
echo "epoch $last_epoch finished uploading, exiting"
exit 0
Expand All @@ -300,16 +306,16 @@ main() {
# Take snapshot and upload to gcloud
# ---------------------------------------------------------------------------

snapshot_file=$(get_snapshot_filename "$SNAPSHOT_DIR" "$previous_epoch_final_slot")
snapshot_path="$SNAPSHOT_DIR/$snapshot_file"
snapshot_file=$(get_snapshot_filename "$SNAPSHOT_OUTPUT_DIR" "$previous_epoch_final_slot")
snapshot_path="$SNAPSHOT_OUTPUT_DIR/$snapshot_file"
if [[ ! -f $snapshot_path ]]; then
post_slack_message "$SLACK_APP_TOKEN" "$SLACK_CHANNEL" "creating snapshot for epoch: $last_epoch slot: $previous_epoch_final_slot"
prune_old_snapshots
create_snapshot_for_slot "$previous_epoch_final_slot" "$SNAPSHOT_DIR" "$LEDGER_DIR"
create_snapshot_for_slot "$previous_epoch_final_slot" "$SNAPSHOT_OUTPUT_DIR" "$SNAPSHOT_ARCHIVE_DIR" "$LEDGER_DIR"

# snapshot file should exist now, grab the filename here (snapshot-$slot-$hash.tar.zst)
snapshot_file=$(get_snapshot_filename "$SNAPSHOT_DIR" "$previous_epoch_final_slot")
snapshot_path="$SNAPSHOT_DIR/$snapshot_file"
snapshot_file=$(get_snapshot_filename "$SNAPSHOT_OUTPUT_DIR" "$previous_epoch_final_slot")
snapshot_path="$SNAPSHOT_OUTPUT_DIR/$snapshot_file"
else
echo "snapshot file already exists at: $snapshot_path"
fi
Expand All @@ -330,10 +336,10 @@ main() {
# ---------------------------------------------------------------------------

stake_meta_filename=stake-meta-"$previous_epoch_final_slot".json
stake_meta_filepath="$SNAPSHOT_DIR/$stake_meta_filename"
stake_meta_filepath="$SNAPSHOT_OUTPUT_DIR/$stake_meta_filename"
if [[ ! -f $stake_meta_filepath ]]; then
post_slack_message "$SLACK_APP_TOKEN" "$SLACK_CHANNEL" "running stake-meta-generator for epoch: $last_epoch slot: $previous_epoch_final_slot"
generate_stake_meta "$previous_epoch_final_slot" "$SNAPSHOT_DIR" "$stake_meta_filename" "$TIP_DISTRIBUTION_PROGRAM_ID" "$TIP_PAYMENT_PROGRAM_ID"
generate_stake_meta "$previous_epoch_final_slot" "$SNAPSHOT_OUTPUT_DIR" "$stake_meta_filename" "$TIP_DISTRIBUTION_PROGRAM_ID" "$TIP_PAYMENT_PROGRAM_ID"
else
echo "stake-meta already exists at: $stake_meta_filepath"
fi
Expand All @@ -353,7 +359,7 @@ main() {
# ---------------------------------------------------------------------------

merkle_tree_filename=merkle-tree-"$previous_epoch_final_slot".json
merkle_tree_filepath="$SNAPSHOT_DIR/$merkle_tree_filename"
merkle_tree_filepath="$SNAPSHOT_OUTPUT_DIR/$merkle_tree_filename"
if [[ ! -f $merkle_tree_filepath ]]; then
post_slack_message "$SLACK_APP_TOKEN" "$SLACK_CHANNEL" "running merkle-root-generator for epoch: $last_epoch slot: $previous_epoch_final_slot"
generate_merkle_trees "$stake_meta_filepath" "$merkle_tree_filepath" "$RPC_URL"
Expand Down Expand Up @@ -399,7 +405,7 @@ main() {
# ---------------------------------------------------------------------------
prune_old_snapshots

touch "$SNAPSHOT_DIR/$last_epoch.done"
touch "$SNAPSHOT_OUTPUT_DIR/$last_epoch.done"
}

main
Loading