From e66572a8911828c1fd4ff34f874c863ff9288006 Mon Sep 17 00:00:00 2001 From: evangriffiths Date: Thu, 19 Sep 2024 13:01:43 +0100 Subject: [PATCH 1/9] Compare agent bet histories with kelly strategy --- .../match_bets_with_langfuse_traces.py | 153 +++++++++++++----- 1 file changed, 115 insertions(+), 38 deletions(-) diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index 07c07390..f58bfcef 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -1,57 +1,134 @@ from datetime import datetime from langfuse import Langfuse -from web3 import Web3 +from pydantic import BaseModel from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.markets.data_models import ResolvedBet from prediction_market_agent_tooling.markets.omen.omen import OmenAgentMarket +from prediction_market_agent_tooling.tools.betting_strategies.kelly_criterion import ( + get_kelly_bet_full, +) from prediction_market_agent_tooling.tools.langfuse_client_utils import ( ProcessMarketTrace, ResolvedBetWithTrace, get_trace_for_bet, get_traces_for_agent, ) +from prediction_market_agent_tooling.tools.utils import check_not_none -if __name__ == "__main__": - api_keys = APIKeys() - assert api_keys.bet_from_address == Web3.to_checksum_address( - "0xe7aa88a1d044e5c987ecce55ae8d2b562a41b72d" # prophetgpt4 - ) - start_time = datetime(2024, 9, 13) - langfuse = Langfuse( - secret_key=api_keys.langfuse_secret_key.get_secret_value(), - public_key=api_keys.langfuse_public_key, - host=api_keys.langfuse_host, - ) - traces = get_traces_for_agent( - agent_name="DeployablePredictionProphetGPT4TurboFinalAgent", - trace_name="process_market", - from_timestamp=start_time, - has_output=True, - client=langfuse, +class KellyBetOutcome(BaseModel): + size: float + direction: bool + correct: bool + profit: float + + +def get_kelly_bet_outcome_for_trace( + trace: ProcessMarketTrace, market_outcome: bool +) -> KellyBetOutcome: + market = trace.market + answer = trace.answer + outcome_token_pool = check_not_none(market.outcome_token_pool) + + kelly_bet = get_kelly_bet_full( + yes_outcome_pool_size=outcome_token_pool[ + market.get_outcome_str_from_bool(True) + ], + no_outcome_pool_size=outcome_token_pool[ + market.get_outcome_str_from_bool(False) + ], + estimated_p_yes=answer.p_yes, + confidence=answer.confidence, + max_bet=2.0, + fee=market.fee, ) - print(f"All traces: {len(traces)}") - process_market_traces = [] - for trace in traces: - if process_market_trace := ProcessMarketTrace.from_langfuse_trace(trace): - process_market_traces.append(process_market_trace) - print(f"All process_market_traces: {len(process_market_traces)}") - - bets: list[ResolvedBet] = OmenAgentMarket.get_resolved_bets_made_since( - better_address=api_keys.bet_from_address, - start_time=start_time, - end_time=None, + received_outcome_tokens = market.get_buy_token_amount( + bet_amount=market.get_bet_amount(kelly_bet.size), + direction=kelly_bet.direction, + ).amount + correct = kelly_bet.direction == market_outcome + profit = received_outcome_tokens - kelly_bet.size if correct else -kelly_bet.size + return KellyBetOutcome( + size=kelly_bet.size, + direction=kelly_bet.direction, + correct=correct, + profit=profit, ) - print(f"All bets: {len(bets)}") - # All bets should have a trace, but not all traces should have a bet - # (e.g. if all markets are deemed unpredictable), so iterate over bets - bets_with_traces: list[ResolvedBetWithTrace] = [] - for bet in bets: - trace = get_trace_for_bet(bet, process_market_traces) - if trace: - bets_with_traces.append(ResolvedBetWithTrace(bet=bet, trace=trace)) - print(f"Matched bets with traces: {len(bets_with_traces)}") +if __name__ == "__main__": + agent_pkey_map = { + "DeployablePredictionProphetGPT4TurboFinalAgent": "...", + "DeployablePredictionProphetGPT4TurboPreviewAgent": "...", + "DeployablePredictionProphetGPT4oAgent": "...", + "DeployableOlasEmbeddingOAAgent": "...", + # "DeployableThinkThoroughlyAgent": "...", # no bets! + # "DeployableThinkThoroughlyProphetResearchAgent": "...", # no bets! + "DeployableKnownOutcomeAgent": "...", + } + print("# Agent Bet vs Theoretical Kelly Bet Comparison") + for agent_name, pkey in agent_pkey_map.items(): + print(f"\n## {agent_name}\n") + api_keys = APIKeys(BET_FROM_PRIVATE_KEY=pkey) + + # Pick a time after pool token number is stored in OmenAgentMarket + start_time = datetime(2024, 9, 13) + + langfuse = Langfuse( + secret_key=api_keys.langfuse_secret_key.get_secret_value(), + public_key=api_keys.langfuse_public_key, + host=api_keys.langfuse_host, + ) + + traces = get_traces_for_agent( + agent_name=agent_name, + trace_name="process_market", + from_timestamp=start_time, + has_output=True, + client=langfuse, + ) + process_market_traces = [] + for trace in traces: + if process_market_trace := ProcessMarketTrace.from_langfuse_trace(trace): + process_market_traces.append(process_market_trace) + + bets: list[ResolvedBet] = OmenAgentMarket.get_resolved_bets_made_since( + better_address=api_keys.bet_from_address, + start_time=start_time, + end_time=None, + ) + print(f"All bets: {len(bets)}") + + # All bets should have a trace, but not all traces should have a bet + # (e.g. if all markets are deemed unpredictable), so iterate over bets + bets_with_traces: list[ResolvedBetWithTrace] = [] + for bet in bets: + trace = get_trace_for_bet(bet, process_market_traces) + if trace: + bets_with_traces.append(ResolvedBetWithTrace(bet=bet, trace=trace)) + + print(f"Matched bets with traces: {len(bets_with_traces)}") + + kelly_bets_outcomes: list[KellyBetOutcome] = [] + for bet_with_trace in bets_with_traces: + bet = bet_with_trace.bet + trace = bet_with_trace.trace + kelly_bet_outcome = get_kelly_bet_outcome_for_trace( + trace=trace, market_outcome=bet.is_correct + ) + kelly_bets_outcomes.append(kelly_bet_outcome) + + total_bet_amount = sum([bt.bet.amount.amount for bt in bets_with_traces]) + total_bet_profit = sum([bt.bet.profit.amount for bt in bets_with_traces]) + total_kelly_amount = sum([kbo.size for kbo in kelly_bets_outcomes]) + total_kelly_profit = sum([kbo.profit for kbo in kelly_bets_outcomes]) + roi = 100 * total_bet_profit / total_bet_amount + kelly_roi = 100 * total_kelly_profit / total_kelly_amount + print( + f"Actual Bet: ROI={roi:.2f}%, amount={total_bet_amount:.2f}, profit={total_bet_profit:.2f}" + ) + print( + f"Kelly Bet: ROI={kelly_roi:.2f}%, amount={total_kelly_amount:.2f}, profit={total_kelly_profit:.2f}" + ) From e48aea3f4b1635650f8d2727f5c8119572c786da Mon Sep 17 00:00:00 2001 From: evangriffiths Date: Thu, 19 Sep 2024 15:19:46 +0100 Subject: [PATCH 2/9] bugfix --- .../monitor/match_bets_with_langfuse_traces.py | 18 ++++++++++++++---- .../tools/langfuse_client_utils.py | 6 ------ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index f58bfcef..18ea62ce 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -89,7 +89,7 @@ def get_kelly_bet_outcome_for_trace( has_output=True, client=langfuse, ) - process_market_traces = [] + process_market_traces: list[ProcessMarketTrace] = [] for trace in traces: if process_market_trace := ProcessMarketTrace.from_langfuse_trace(trace): process_market_traces.append(process_market_trace) @@ -99,7 +99,6 @@ def get_kelly_bet_outcome_for_trace( start_time=start_time, end_time=None, ) - print(f"All bets: {len(bets)}") # All bets should have a trace, but not all traces should have a bet # (e.g. if all markets are deemed unpredictable), so iterate over bets @@ -109,17 +108,28 @@ def get_kelly_bet_outcome_for_trace( if trace: bets_with_traces.append(ResolvedBetWithTrace(bet=bet, trace=trace)) - print(f"Matched bets with traces: {len(bets_with_traces)}") + print(f"Number of bets since {start_time}: {len(bets_with_traces)}") + if len(bets_with_traces) != len(bets): + raise ValueError( + f"{len(bets) - len(bets_with_traces)} bets do not have a corresponding trace" + ) kelly_bets_outcomes: list[KellyBetOutcome] = [] for bet_with_trace in bets_with_traces: bet = bet_with_trace.bet trace = bet_with_trace.trace kelly_bet_outcome = get_kelly_bet_outcome_for_trace( - trace=trace, market_outcome=bet.is_correct + trace=trace, market_outcome=bet.market_outcome ) kelly_bets_outcomes.append(kelly_bet_outcome) + ## Uncomment for debug + # print( + # f"Actual: size={bet.amount.amount:.2f}, dir={bet.outcome}, correct={bet.is_correct} profit={bet.profit.amount:.2f} | " + # f"Kelly: size={kelly_bet_outcome.size:.2f}, dir={kelly_bet_outcome.direction}, correct={kelly_bet_outcome.correct}, profit={kelly_bet_outcome.profit:.2f} | " + # f"outcome={bet.market_outcome}, mrkt_p_yes={trace.market.current_p_yes:.2f}, est_p_yes={trace.answer.p_yes:.2f}, conf={trace.answer.confidence:.2f}" + # ) + total_bet_amount = sum([bt.bet.amount.amount for bt in bets_with_traces]) total_bet_profit = sum([bt.bet.profit.amount for bt in bets_with_traces]) total_kelly_amount = sum([kbo.size for kbo in kelly_bets_outcomes]) diff --git a/prediction_market_agent_tooling/tools/langfuse_client_utils.py b/prediction_market_agent_tooling/tools/langfuse_client_utils.py index 0f99d75b..f82c0caf 100644 --- a/prediction_market_agent_tooling/tools/langfuse_client_utils.py +++ b/prediction_market_agent_tooling/tools/langfuse_client_utils.py @@ -150,10 +150,4 @@ def get_trace_for_bet( add_utc_timezone_validator(bet.created_time), [t.timestamp for t in traces_for_bet], ) - # Sanity check - the trace should be after the bet - if traces_for_bet[closest_trace_index].timestamp < add_utc_timezone_validator( - bet.created_time - ): - return None - return traces_for_bet[closest_trace_index] From 48e8993e1cac8a84a2ff3179016db8e46f33fcab Mon Sep 17 00:00:00 2001 From: evangriffiths Date: Thu, 19 Sep 2024 15:21:03 +0100 Subject: [PATCH 3/9] mypy --- examples/monitor/match_bets_with_langfuse_traces.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index 18ea62ce..64336b48 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from prediction_market_agent_tooling.config import APIKeys +from prediction_market_agent_tooling.gtypes import PrivateKey from prediction_market_agent_tooling.markets.data_models import ResolvedBet from prediction_market_agent_tooling.markets.omen.omen import OmenAgentMarket from prediction_market_agent_tooling.tools.betting_strategies.kelly_criterion import ( @@ -71,7 +72,7 @@ def get_kelly_bet_outcome_for_trace( print("# Agent Bet vs Theoretical Kelly Bet Comparison") for agent_name, pkey in agent_pkey_map.items(): print(f"\n## {agent_name}\n") - api_keys = APIKeys(BET_FROM_PRIVATE_KEY=pkey) + api_keys = APIKeys(BET_FROM_PRIVATE_KEY=PrivateKey(pkey)) # Pick a time after pool token number is stored in OmenAgentMarket start_time = datetime(2024, 9, 13) From 2325c4bd3b915f3d2a2bea049f763001161c633c Mon Sep 17 00:00:00 2001 From: evangriffiths Date: Thu, 19 Sep 2024 15:30:06 +0100 Subject: [PATCH 4/9] mypy --- examples/monitor/match_bets_with_langfuse_traces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index 64336b48..147a87c4 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -1,7 +1,7 @@ from datetime import datetime from langfuse import Langfuse -from pydantic import BaseModel +from pydantic import BaseModel, SecretStr from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.gtypes import PrivateKey @@ -72,7 +72,7 @@ def get_kelly_bet_outcome_for_trace( print("# Agent Bet vs Theoretical Kelly Bet Comparison") for agent_name, pkey in agent_pkey_map.items(): print(f"\n## {agent_name}\n") - api_keys = APIKeys(BET_FROM_PRIVATE_KEY=PrivateKey(pkey)) + api_keys = APIKeys(BET_FROM_PRIVATE_KEY=PrivateKey(SecretStr(pkey))) # Pick a time after pool token number is stored in OmenAgentMarket start_time = datetime(2024, 9, 13) From b78489fa8b657bdcc712706dc91038c90d1b548a Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 20 Sep 2024 10:12:57 +0200 Subject: [PATCH 5/9] Use dynamic max bet amount --- .../monitor/match_bets_with_langfuse_traces.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index 147a87c4..0de2a993 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -27,7 +27,7 @@ class KellyBetOutcome(BaseModel): def get_kelly_bet_outcome_for_trace( - trace: ProcessMarketTrace, market_outcome: bool + trace: ProcessMarketTrace, market_outcome: bool, max_bet: float ) -> KellyBetOutcome: market = trace.market answer = trace.answer @@ -42,7 +42,7 @@ def get_kelly_bet_outcome_for_trace( ], estimated_p_yes=answer.p_yes, confidence=answer.confidence, - max_bet=2.0, + max_bet=max_bet, fee=market.fee, ) received_outcome_tokens = market.get_buy_token_amount( @@ -115,14 +115,23 @@ def get_kelly_bet_outcome_for_trace( f"{len(bets) - len(bets_with_traces)} bets do not have a corresponding trace" ) + # "Born" agent with initial funding, simulate as if he was doing bets one by one. + agent_balance = 50.0 + kelly_bets_outcomes: list[KellyBetOutcome] = [] for bet_with_trace in bets_with_traces: + if agent_balance <= 0: + print(f"Agent died with balance {agent_balance}.") + break bet = bet_with_trace.bet trace = bet_with_trace.trace kelly_bet_outcome = get_kelly_bet_outcome_for_trace( - trace=trace, market_outcome=bet.market_outcome + trace=trace, + market_outcome=bet.market_outcome, + max_bet=agent_balance * 0.9, ) kelly_bets_outcomes.append(kelly_bet_outcome) + agent_balance += kelly_bet_outcome.profit ## Uncomment for debug # print( From 0ab2ef511d77dbd0d52fbf97c61a105643a3d856 Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Fri, 20 Sep 2024 10:21:41 +0200 Subject: [PATCH 6/9] print final balance --- examples/monitor/match_bets_with_langfuse_traces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index 0de2a993..48832786 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -150,5 +150,5 @@ def get_kelly_bet_outcome_for_trace( f"Actual Bet: ROI={roi:.2f}%, amount={total_bet_amount:.2f}, profit={total_bet_profit:.2f}" ) print( - f"Kelly Bet: ROI={kelly_roi:.2f}%, amount={total_kelly_amount:.2f}, profit={total_kelly_profit:.2f}" + f"Kelly Bet: ROI={kelly_roi:.2f}%, amount={total_kelly_amount:.2f}, profit={total_kelly_profit:.2f}, final agent balance: {agent_balance:.2f}" ) From 1212dcac61cca946ab776b3462b0fe4a7d50646e Mon Sep 17 00:00:00 2001 From: evangriffiths Date: Fri, 20 Sep 2024 10:19:35 +0100 Subject: [PATCH 7/9] comment out debug print --- examples/monitor/match_bets_with_langfuse_traces.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index 405b3a53..6f8a0ee4 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -134,12 +134,12 @@ def get_kelly_bet_outcome_for_trace( kelly_bets_outcomes.append(kelly_bet_outcome) agent_balance += kelly_bet_outcome.profit - # Uncomment for debug - print( - f"Actual: size={bet.amount.amount:.2f}, dir={bet.outcome}, correct={bet.is_correct} profit={bet.profit.amount:.2f} | " - f"Kelly: size={kelly_bet_outcome.size:.2f}, dir={kelly_bet_outcome.direction}, correct={kelly_bet_outcome.correct}, profit={kelly_bet_outcome.profit:.2f} | " - f"outcome={bet.market_outcome}, mrkt_p_yes={trace.market.current_p_yes:.2f}, est_p_yes={trace.answer.p_yes:.2f}, conf={trace.answer.confidence:.2f}" - ) + # # Uncomment for debug + # print( + # f"Actual: size={bet.amount.amount:.2f}, dir={bet.outcome}, correct={bet.is_correct} profit={bet.profit.amount:.2f} | " + # f"Kelly: size={kelly_bet_outcome.size:.2f}, dir={kelly_bet_outcome.direction}, correct={kelly_bet_outcome.correct}, profit={kelly_bet_outcome.profit:.2f} | " + # f"outcome={bet.market_outcome}, mrkt_p_yes={trace.market.current_p_yes:.2f}, est_p_yes={trace.answer.p_yes:.2f}, conf={trace.answer.confidence:.2f}" + # ) total_bet_amount = sum([bt.bet.amount.amount for bt in bets_with_traces]) total_bet_profit = sum([bt.bet.profit.amount for bt in bets_with_traces]) From c6ee7fe652fc9393b95a45a3a3831282289d2d4e Mon Sep 17 00:00:00 2001 From: evangriffiths Date: Fri, 20 Sep 2024 13:30:25 +0100 Subject: [PATCH 8/9] Add util function to get pkeys from GCP --- .../match_bets_with_langfuse_traces.py | 30 +++++++++++-------- .../tools/utils.py | 18 +++++++++++ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index 6f8a0ee4..cbf326c2 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -1,10 +1,9 @@ from datetime import datetime from langfuse import Langfuse -from pydantic import BaseModel, SecretStr +from pydantic import BaseModel from prediction_market_agent_tooling.config import APIKeys -from prediction_market_agent_tooling.gtypes import PrivateKey from prediction_market_agent_tooling.markets.data_models import ResolvedBet from prediction_market_agent_tooling.markets.omen.omen import OmenAgentMarket from prediction_market_agent_tooling.tools.betting_strategies.kelly_criterion import ( @@ -16,7 +15,10 @@ get_trace_for_bet, get_traces_for_agent, ) -from prediction_market_agent_tooling.tools.utils import check_not_none +from prediction_market_agent_tooling.tools.utils import ( + check_not_none, + get_private_key_from_gcp_secret, +) class KellyBetOutcome(BaseModel): @@ -60,20 +62,24 @@ def get_kelly_bet_outcome_for_trace( if __name__ == "__main__": + # Get the private keys for the agents from GCP Secret Manager + agent_gcp_secret_map = { + "DeployablePredictionProphetGPT4TurboFinalAgent": "pma-prophetgpt4turbo-final", + "DeployablePredictionProphetGPT4TurboPreviewAgent": "pma-prophetgpt4", + "DeployablePredictionProphetGPT4oAgent": "pma-prophetgpt3", + "DeployableOlasEmbeddingOAAgent": "pma-evo-olas-embeddingoa", + # "DeployableThinkThoroughlyAgent": "pma-think-thoroughly", # no bets! + # "DeployableThinkThoroughlyProphetResearchAgent": "pma-think-thoroughly-prophet-research", # no bets! + "DeployableKnownOutcomeAgent": "pma-knownoutcome", + } agent_pkey_map = { - "DeployablePredictionProphetGPT4TurboFinalAgent": "...", - "DeployablePredictionProphetGPT4TurboPreviewAgent": "...", - "DeployablePredictionProphetGPT4oAgent": "...", - "DeployableOlasEmbeddingOAAgent": "...", - # "DeployableThinkThoroughlyAgent": "...", # no bets! - # "DeployableThinkThoroughlyProphetResearchAgent": "...", # no bets! - "DeployableKnownOutcomeAgent": "...", + k: get_private_key_from_gcp_secret(v) for k, v in agent_gcp_secret_map.items() } print("# Agent Bet vs Theoretical Kelly Bet Comparison") - for agent_name, pkey in agent_pkey_map.items(): + for agent_name, private_key in agent_pkey_map.items(): print(f"\n## {agent_name}\n") - api_keys = APIKeys(BET_FROM_PRIVATE_KEY=PrivateKey(SecretStr(pkey))) + api_keys = APIKeys(BET_FROM_PRIVATE_KEY=private_key) # Pick a time after pool token number is stored in OmenAgentMarket start_time = datetime(2024, 9, 13) diff --git a/prediction_market_agent_tooling/tools/utils.py b/prediction_market_agent_tooling/tools/utils.py index 2351fe70..141f1b3c 100644 --- a/prediction_market_agent_tooling/tools/utils.py +++ b/prediction_market_agent_tooling/tools/utils.py @@ -1,3 +1,4 @@ +import json import os import subprocess import typing as t @@ -6,12 +7,14 @@ import pytz import requests +from google.cloud import secretmanager from pydantic import BaseModel, ValidationError from scipy.optimize import newton from scipy.stats import entropy from prediction_market_agent_tooling.gtypes import ( DatetimeWithTimezone, + PrivateKey, Probability, SecretStr, ) @@ -210,3 +213,18 @@ def f(r: float) -> float: amount_to_sell = newton(f, 0) return float(amount_to_sell) * 0.999999 # Avoid rounding errors + + +def get_private_key_from_gcp_secret( + secret_id: str, + project_id: str = "582587111398", # Gnosis AI default project_id + version_id: str = "latest", +) -> PrivateKey: + client = secretmanager.SecretManagerServiceClient() + name = f"projects/{project_id}/secrets/{secret_id}/versions/{version_id}" + response = client.access_secret_version(request={"name": name}) + secret_payload = response.payload.data.decode("UTF-8") + secret_json = json.loads(secret_payload) + if "private_key" not in secret_json: + raise ValueError(f"Private key not found in gcp secret {secret_id}") + return PrivateKey(SecretStr(secret_json["private_key"])) From 82c103b45c29d212be553a3466d880aa5596587c Mon Sep 17 00:00:00 2001 From: Peter Jung Date: Mon, 23 Sep 2024 15:38:44 +0200 Subject: [PATCH 9/9] Simulate all strategies we have with different bet sizes (#424) --- .../match_bets_with_langfuse_traces.py | 189 +++++++++++------- .../deploy/betting_strategy.py | 67 +++++++ 2 files changed, 186 insertions(+), 70 deletions(-) diff --git a/examples/monitor/match_bets_with_langfuse_traces.py b/examples/monitor/match_bets_with_langfuse_traces.py index cbf326c2..494c854a 100644 --- a/examples/monitor/match_bets_with_langfuse_traces.py +++ b/examples/monitor/match_bets_with_langfuse_traces.py @@ -1,61 +1,80 @@ from datetime import datetime +from typing import Any +import pandas as pd from langfuse import Langfuse from pydantic import BaseModel from prediction_market_agent_tooling.config import APIKeys +from prediction_market_agent_tooling.deploy.betting_strategy import ( + BettingStrategy, + KellyBettingStrategy, + MaxAccuracyBettingStrategy, + MaxAccuracyWithKellyScaledBetsStrategy, + MaxExpectedValueBettingStrategy, + ProbabilisticAnswer, + TradeType, +) from prediction_market_agent_tooling.markets.data_models import ResolvedBet from prediction_market_agent_tooling.markets.omen.omen import OmenAgentMarket -from prediction_market_agent_tooling.tools.betting_strategies.kelly_criterion import ( - get_kelly_bet_full, -) from prediction_market_agent_tooling.tools.langfuse_client_utils import ( ProcessMarketTrace, ResolvedBetWithTrace, get_trace_for_bet, get_traces_for_agent, ) -from prediction_market_agent_tooling.tools.utils import ( - check_not_none, - get_private_key_from_gcp_secret, -) +from prediction_market_agent_tooling.tools.utils import get_private_key_from_gcp_secret -class KellyBetOutcome(BaseModel): +class SimulatedOutcome(BaseModel): size: float direction: bool correct: bool profit: float -def get_kelly_bet_outcome_for_trace( - trace: ProcessMarketTrace, market_outcome: bool, max_bet: float -) -> KellyBetOutcome: +def get_outcome_for_trace( + strategy: BettingStrategy, + trace: ProcessMarketTrace, + market_outcome: bool, +) -> SimulatedOutcome | None: market = trace.market answer = trace.answer - outcome_token_pool = check_not_none(market.outcome_token_pool) - - kelly_bet = get_kelly_bet_full( - yes_outcome_pool_size=outcome_token_pool[ - market.get_outcome_str_from_bool(True) - ], - no_outcome_pool_size=outcome_token_pool[ - market.get_outcome_str_from_bool(False) - ], - estimated_p_yes=answer.p_yes, - confidence=answer.confidence, - max_bet=max_bet, - fee=market.fee, + + trades = strategy.calculate_trades( + existing_position=None, + answer=ProbabilisticAnswer( + p_yes=answer.p_yes, + confidence=answer.confidence, + ), + market=market, ) + # For example, when our predicted p_yes is 95%, but market is already trading at 99%, and we don't have anything to sell, Kelly will yield no trades. + if not trades: + return None + assert ( + len(trades) == 1 + ), f"Should be always one trade if no existing position is given: {trades=}; {answer=}; {market=}" + assert ( + trades[0].trade_type == TradeType.BUY + ), "Can only buy without previous position." + buy_trade = trades[0] + received_outcome_tokens = market.get_buy_token_amount( - bet_amount=market.get_bet_amount(kelly_bet.size), - direction=kelly_bet.direction, + bet_amount=market.get_bet_amount(buy_trade.amount.amount), + direction=buy_trade.outcome, ).amount - correct = kelly_bet.direction == market_outcome - profit = received_outcome_tokens - kelly_bet.size if correct else -kelly_bet.size - return KellyBetOutcome( - size=kelly_bet.size, - direction=kelly_bet.direction, + + correct = buy_trade.outcome == market_outcome + profit = ( + received_outcome_tokens - buy_trade.amount.amount + if correct + else -buy_trade.amount.amount + ) + + return SimulatedOutcome( + size=buy_trade.amount.amount, + direction=buy_trade.outcome, correct=correct, profit=profit, ) @@ -75,8 +94,23 @@ def get_kelly_bet_outcome_for_trace( agent_pkey_map = { k: get_private_key_from_gcp_secret(v) for k, v in agent_gcp_secret_map.items() } - - print("# Agent Bet vs Theoretical Kelly Bet Comparison") + # Define strategies we want to test out + strategies = [ + MaxAccuracyBettingStrategy(bet_amount=1), + MaxAccuracyBettingStrategy(bet_amount=2), + MaxAccuracyBettingStrategy(bet_amount=25), + KellyBettingStrategy(max_bet_amount=1), + KellyBettingStrategy(max_bet_amount=2), + KellyBettingStrategy(max_bet_amount=25), + MaxAccuracyWithKellyScaledBetsStrategy(max_bet_amount=1), + MaxAccuracyWithKellyScaledBetsStrategy(max_bet_amount=2), + MaxAccuracyWithKellyScaledBetsStrategy(max_bet_amount=25), + MaxExpectedValueBettingStrategy(bet_amount=1), + MaxExpectedValueBettingStrategy(bet_amount=2), + MaxExpectedValueBettingStrategy(bet_amount=25), + ] + + print("# Agent Bet vs Simulated Bet Comparison") for agent_name, private_key in agent_pkey_map.items(): print(f"\n## {agent_name}\n") api_keys = APIKeys(BET_FROM_PRIVATE_KEY=private_key) @@ -116,46 +150,61 @@ def get_kelly_bet_outcome_for_trace( if trace: bets_with_traces.append(ResolvedBetWithTrace(bet=bet, trace=trace)) - print(f"Number of bets since {start_time}: {len(bets_with_traces)}") + print(f"Number of bets since {start_time}: {len(bets_with_traces)}\n") if len(bets_with_traces) != len(bets): raise ValueError( f"{len(bets) - len(bets_with_traces)} bets do not have a corresponding trace" ) - # "Born" agent with initial funding, simulate as if he was doing bets one by one. - agent_balance = 50.0 - - kelly_bets_outcomes: list[KellyBetOutcome] = [] - for bet_with_trace in bets_with_traces: - if agent_balance <= 0: - print(f"Agent died with balance {agent_balance}.") - break - bet = bet_with_trace.bet - trace = bet_with_trace.trace - kelly_bet_outcome = get_kelly_bet_outcome_for_trace( - trace=trace, - market_outcome=bet.market_outcome, - max_bet=agent_balance * 0.9, + simulations: list[dict[str, Any]] = [] + + for strategy_idx, strategy in enumerate(strategies): + # "Born" agent with initial funding, simulate as if he was doing bets one by one. + starting_balance = 50.0 + agent_balance = starting_balance + simulated_outcomes: list[SimulatedOutcome] = [] + + for bet_with_trace in bets_with_traces: + bet = bet_with_trace.bet + trace = bet_with_trace.trace + simulated_outcome = get_outcome_for_trace( + strategy=strategy, trace=trace, market_outcome=bet.market_outcome + ) + if simulated_outcome is None: + continue + simulated_outcomes.append(simulated_outcome) + agent_balance += simulated_outcome.profit + + total_bet_amount = sum([bt.bet.amount.amount for bt in bets_with_traces]) + total_bet_profit = sum([bt.bet.profit.amount for bt in bets_with_traces]) + total_simulated_amount = sum([so.size for so in simulated_outcomes]) + total_simulated_profit = sum([so.profit for so in simulated_outcomes]) + roi = 100 * total_bet_profit / total_bet_amount + simulated_roi = 100 * total_simulated_profit / total_simulated_amount + + # At the beginning, add also the agent's current strategy. + if strategy_idx == 0: + simulations.append( + { + "strategy": "original", + "bet_amount": total_bet_amount, + "bet_profit": total_bet_profit, + "roi": roi, + # We don't know these for the original run. + "start_balance": None, + "end_balance": None, + } + ) + + simulations.append( + { + "strategy": repr(strategy), + "bet_amount": total_simulated_amount, + "bet_profit": total_simulated_profit, + "roi": simulated_roi, + "start_balance": starting_balance, + "end_balance": agent_balance, + } ) - kelly_bets_outcomes.append(kelly_bet_outcome) - agent_balance += kelly_bet_outcome.profit - - # # Uncomment for debug - # print( - # f"Actual: size={bet.amount.amount:.2f}, dir={bet.outcome}, correct={bet.is_correct} profit={bet.profit.amount:.2f} | " - # f"Kelly: size={kelly_bet_outcome.size:.2f}, dir={kelly_bet_outcome.direction}, correct={kelly_bet_outcome.correct}, profit={kelly_bet_outcome.profit:.2f} | " - # f"outcome={bet.market_outcome}, mrkt_p_yes={trace.market.current_p_yes:.2f}, est_p_yes={trace.answer.p_yes:.2f}, conf={trace.answer.confidence:.2f}" - # ) - - total_bet_amount = sum([bt.bet.amount.amount for bt in bets_with_traces]) - total_bet_profit = sum([bt.bet.profit.amount for bt in bets_with_traces]) - total_kelly_amount = sum([kbo.size for kbo in kelly_bets_outcomes]) - total_kelly_profit = sum([kbo.profit for kbo in kelly_bets_outcomes]) - roi = 100 * total_bet_profit / total_bet_amount - kelly_roi = 100 * total_kelly_profit / total_kelly_amount - print( - f"Actual Bet: ROI={roi:.2f}%, amount={total_bet_amount:.2f}, profit={total_bet_profit:.2f}" - ) - print( - f"Kelly Bet: ROI={kelly_roi:.2f}%, amount={total_kelly_amount:.2f}, profit={total_kelly_profit:.2f}, final agent balance: {agent_balance:.2f}" - ) + + print(pd.DataFrame.from_records(simulations).to_markdown(index=False)) diff --git a/prediction_market_agent_tooling/deploy/betting_strategy.py b/prediction_market_agent_tooling/deploy/betting_strategy.py index 481bc8b0..18af7183 100644 --- a/prediction_market_agent_tooling/deploy/betting_strategy.py +++ b/prediction_market_agent_tooling/deploy/betting_strategy.py @@ -141,6 +141,9 @@ def calculate_trades( def calculate_direction(market_p_yes: float, estimate_p_yes: float) -> bool: return estimate_p_yes >= 0.5 + def __repr__(self) -> str: + return f"{self.__class__.__name__}(bet_amount={self.bet_amount})" + class MaxExpectedValueBettingStrategy(MaxAccuracyBettingStrategy): @staticmethod @@ -202,3 +205,67 @@ def calculate_trades( existing_position, target_position, market=market ) return trades + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(max_bet_amount={self.max_bet_amount})" + + +class MaxAccuracyWithKellyScaledBetsStrategy(BettingStrategy): + def __init__(self, max_bet_amount: float = 10): + self.max_bet_amount = max_bet_amount + + def adjust_bet_amount( + self, existing_position: Position | None, market: AgentMarket + ) -> float: + existing_position_total_amount = ( + existing_position.total_amount.amount if existing_position else 0 + ) + return self.max_bet_amount + existing_position_total_amount + + def calculate_trades( + self, + existing_position: Position | None, + answer: ProbabilisticAnswer, + market: AgentMarket, + ) -> list[Trade]: + adjusted_bet_amount = self.adjust_bet_amount(existing_position, market) + outcome_token_pool = check_not_none(market.outcome_token_pool) + + # Fixed direction of bet, only use Kelly to adjust the bet size based on market's outcome pool size. + estimated_p_yes = float(answer.p_yes > 0.5) + confidence = 1.0 + + kelly_bet = ( + get_kelly_bet_full( + yes_outcome_pool_size=outcome_token_pool[ + market.get_outcome_str_from_bool(True) + ], + no_outcome_pool_size=outcome_token_pool[ + market.get_outcome_str_from_bool(False) + ], + estimated_p_yes=estimated_p_yes, + max_bet=adjusted_bet_amount, + confidence=confidence, + ) + if market.has_token_pool() + else get_kelly_bet_simplified( + adjusted_bet_amount, + market.current_p_yes, + estimated_p_yes, + confidence, + ) + ) + + amounts = { + market.get_outcome_str_from_bool(kelly_bet.direction): TokenAmount( + amount=kelly_bet.size, currency=market.currency + ), + } + target_position = Position(market_id=market.id, amounts=amounts) + trades = self._build_rebalance_trades_from_positions( + existing_position, target_position, market=market + ) + return trades + + def __repr__(self) -> str: + return f"{self.__class__.__name__}(max_bet_amount={self.max_bet_amount})"