diff --git a/ccc/tests/test_utils.py b/ccc/tests/test_utils.py index d4823224..80a58dd2 100644 --- a/ccc/tests/test_utils.py +++ b/ccc/tests/test_utils.py @@ -145,7 +145,6 @@ def test_json_to_dict_exception(): "df,output_type,precision", test_data, ids=["tex", "json", "html"] ) def test_save_return_table(df, output_type, precision): - test_str = utils.save_return_table(df, output_type, None, precision) assert isinstance(test_str, str) diff --git a/cs-config/cs_config/functions.py b/cs-config/cs_config/functions.py index b396fe2c..c66a4a32 100644 --- a/cs-config/cs_config/functions.py +++ b/cs-config/cs_config/functions.py @@ -11,8 +11,11 @@ from .helpers import retrieve_puf import cs2tc -AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", "") -AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "") +AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID") +AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY") +PUF_S3_FILE_LOCATION = os.environ.get( + "PUF_S3_LOCATION", "s3://ospc-data-files/puf.20210720.csv.gz" +) class MetaParams(paramtools.Parameters): @@ -174,7 +177,9 @@ def run_model(meta_param_dict, adjustment): meta_params.adjust(meta_param_dict) # Get data chosen by user if meta_params.data_source == "PUF": - data = retrieve_puf(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + data = retrieve_puf( + PUF_S3_FILE_LOCATION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY + ) else: data = "cps" # Get TC params adjustments diff --git a/cs-config/cs_config/helpers.py b/cs-config/cs_config/helpers.py index dac7d0cd..1dfc1335 100644 --- a/cs-config/cs_config/helpers.py +++ b/cs-config/cs_config/helpers.py @@ -5,18 +5,20 @@ import os from pathlib import Path import warnings +import os try: from s3fs import S3FileSystem -except ImportError: +except ImportError as ie: S3FileSystem = None import pandas as pd from ccc.utils import TC_LAST_YEAR -AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID") -AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY") - -PUF_S3_FILE_NAME = "puf.20210720.csv.gz" +AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", None) +AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", None) +PUF_S3_FILE_LOCATION = os.environ.get( + "PUF_S3_LOCATION", "s3://ospc-data-files/puf.20210720.csv.gz" +) POLICY_SCHEMA = { "labels": { @@ -79,6 +81,7 @@ def retrieve_puf( + puf_s3_file_location=PUF_S3_FILE_LOCATION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, ): @@ -89,15 +92,15 @@ def retrieve_puf( has_credentials = ( aws_access_key_id is not None and aws_secret_access_key is not None ) - if has_credentials and s3_reader_installed: - print("Reading puf from S3 bucket.") + if puf_s3_file_location and has_credentials and s3_reader_installed: + print("Reading puf from S3 bucket.", puf_s3_file_location) fs = S3FileSystem( key=AWS_ACCESS_KEY_ID, secret=AWS_SECRET_ACCESS_KEY, ) - with fs.open(f"s3://ospc-data-files/{PUF_S3_FILE_NAME}") as f: + with fs.open(puf_s3_file_location) as f: # Skips over header from top of file. - puf_df = pd.read_csv(f, compression="gzip") + puf_df = pd.read_csv(f) return puf_df elif Path("puf.csv.gz").exists(): print("Reading puf from puf.csv.gz.") @@ -107,7 +110,8 @@ def retrieve_puf( return pd.read_csv("puf.csv") else: warnings.warn( - f"PUF file not available (has_credentials={has_credentials}, " + f"PUF file not available (puf_location={puf_s3_file_location}, " + f"has_credentials={has_credentials}, " f"s3_reader_installed={s3_reader_installed})" ) return None diff --git a/cs-config/install.sh b/cs-config/install.sh index 5c74c541..657b3874 100644 --- a/cs-config/install.sh +++ b/cs-config/install.sh @@ -1 +1,2 @@ -conda install -c conda-forge ccc s3fs +conda install -c conda-forge s3fs +pip install -e . diff --git a/environment.yml b/environment.yml index 1dbe83ab..5807648b 100644 --- a/environment.yml +++ b/environment.yml @@ -5,6 +5,7 @@ dependencies: - python - "taxcalc>=3.1.0" - "paramtools>=0.18.0" +- "pandas>=2.2.0" - bokeh>=3.1.1 - setuptools - pip