Merge pull request #391 from jdebacker/flex_micro_data

Allow for more flexibility in individual income tax micro data input

.github/workflows/build_and_test.yml

@@ -34,9 +34,9 @@ jobs:
         shell: bash -l {0}
         working-directory: ./
         run: |
-          pytest -m 'not needs_puf' --cov=./ --cov-report=xml
+          pytest -m 'not needs_puf and not needs_tmd' --cov=./ --cov-report=xml
       - name: Upload coverage to Codecov
-        if: matrix.os == 'ubuntu-latest'
+        if: matrix.os == 'ubuntu-latest' && contains(github.repository, 'PSLmodels/Cost-of-Capital-Calculator')
         uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos

README.md

@@ -53,4 +53,4 @@ Results will change as the underlying models improve. A fundamental reason for a
 
 
 ## Citing the Cost-of-Capital-Calculator Model
-Cost-of-Capital-Calculator (Version 1.4.0)[Source code], https://github.com/PSLmodels/Cost-of-Capital-Calculator
+Cost-of-Capital-Calculator (Version 1.4.1)[Source code], https://github.com/PSLmodels/Cost-of-Capital-Calculator

ccc/__init__.py

@@ -6,4 +6,4 @@
 from ccc.data import *
 from ccc.calculator import *
 
-__version__ = "1.4.0"
+__version__ = "1.4.1"

ccc/get_taxcalc_rates.py

@@ -1,11 +1,10 @@
 # imports
 import numpy as np
-from taxcalc import Policy, Records, Calculator
+from taxcalc import Policy, Records, Calculator, GrowFactors
 from ccc.utils import DEFAULT_START_YEAR, TC_LAST_YEAR, RECORDS_START_YEAR
 
 
 def get_calculator(
-    baseline,
     calculator_start_year,
     baseline_policy=None,
     reform=None,
@@ -16,14 +15,19 @@ def get_calculator(
 ):
     """
     This function creates the tax calculator object for the microsim
+    model.
+
+    Note: gfactors and weights are only used if provide custom data
+    path or file with those gfactors and weights.  Otherwise, the
+    model defaults to those gfactors and weights from Tax-Calculator.
 
     Args:
-        baseline (bool): `True` if baseline tax policy
         calculator_start_year (integer): first year of budget window
         baseline_policy (dictionary): IIT baseline parameters
         reform (dictionary): IIT reform parameters
         data (string or Pandas DataFrame): path to file or DataFrame
             for Tax-Calculator Records object (optional)
+        gfactors (str or DataFrame): grow factors to extrapolate data
         weights (DataFrame): weights DataFrame for Tax-Calculator
             Records object (optional)
         records_start_year (integer): the start year for the data and
@@ -36,32 +40,36 @@ def get_calculator(
     # create a calculator
     policy1 = Policy()
     if data is not None and "cps" in data:
+        print("Using CPS")
         records1 = Records.cps_constructor()
         # impute short and long term capital gains if using CPS data
         # in 2012 SOI data 6.587% of CG as short-term gains
         records1.p22250 = 0.06587 * records1.e01100
         records1.p23250 = (1 - 0.06587) * records1.e01100
         # set total capital gains to zero
         records1.e01100 = np.zeros(records1.e01100.shape[0])
+    elif data is None or "puf" in data:  # pragma: no cover
+        print("Using PUF")
+        records1 = Records()
+    elif data is not None and "tmd" in data:  # pragma: no cover
+        print("Using TMD")
+        records1 = Records.tmd_constructor("tmd.csv.gz")
     elif data is not None:  # pragma: no cover
+        print("Data is ", data)
+        print("Weights are ", weights)
+        print("Records start year is ", records_start_year)
         records1 = Records(
             data=data,
             gfactors=gfactors,
             weights=weights,
             start_year=records_start_year,
         )  # pragma: no cover
-    else:
-        records1 = Records()  # pragma: no cover
+    else:  # pragma: no cover
+        raise ValueError("Please provide data or use CPS, PUF, or TMD.")
 
-    if baseline:
-        if (
-            baseline_policy
-        ):  # if something other than current law policy baseline
-            update_policy(policy1, baseline_policy)
-
-    if not baseline:
-        if baseline_policy:  # update baseline policy to layer reform on top
-            update_policy(policy1, baseline_policy)
+    if baseline_policy:  # if something other than current law policy baseline
+        update_policy(policy1, baseline_policy)
+    if reform:  # if there is a reform
         update_policy(policy1, reform)
 
     # the default set up increments year to 2013
@@ -79,32 +87,42 @@ def get_calculator(
 
 
 def get_rates(
-    baseline=False,
     start_year=DEFAULT_START_YEAR,
     baseline_policy=None,
     reform={},
     data="cps",
+    gfactors=None,
+    weights=None,
+    records_start_year=RECORDS_START_YEAR,
 ):
     """
     This function computes weighted average marginal tax rates using
     micro data from the tax calculator
 
     Args:
-        baseline (bool): `True` if baseline tax policy, `False` if reform
-        start_year (integer): first year of budget window
+        start_year (integer): start year for the simulations
+        baseline_policy (dict): baseline parameters
         reform (dict): reform parameters
+        data (string or Pandas DataFrame): path to file or DataFrame
+            for Tax-Calculator Records object (optional)
+        gfactors (Tax-Calculator GrowFactors object): grow factors
+        weights (str): path to weights file for Tax-Calculator
+            Records object
+        records_start_year (integer): the start year for the microdata
 
     Returns:
         individual_rates (dict): individual income (IIT+payroll)
             marginal tax rates
 
     """
     calc1 = get_calculator(
-        baseline=baseline,
         calculator_start_year=start_year,
         baseline_policy=baseline_policy,
         reform=reform,
         data=data,
+        gfactors=None,
+        weights=None,
+        records_start_year=records_start_year,
     )
 
     # running all the functions and calculates taxes

ccc/parameters.py

@@ -4,7 +4,7 @@
 
 # import ccc
 from ccc.get_taxcalc_rates import get_rates
-from ccc.utils import DEFAULT_START_YEAR
+from ccc.utils import DEFAULT_START_YEAR, RECORDS_START_YEAR
 import ccc.paramfunctions as pf
 
 CURRENT_PATH = os.path.abspath(os.path.dirname(__file__))
@@ -23,32 +23,53 @@ class Specification(paramtools.Parameters):
     def __init__(
         self,
         test=False,
-        baseline=False,
         year=DEFAULT_START_YEAR,
         call_tc=False,
         baseline_policy=None,
-        iit_reform={},
+        iit_reform=None,
         data="cps",
+        gfactors=None,
+        weights=None,
+        records_start_year=RECORDS_START_YEAR,
     ):
         super().__init__()
         self.set_state(year=year)
         self.test = test
-        self.baseline = baseline
         self.year = year
         self.baseline_policy = baseline_policy
         self.iit_reform = iit_reform
         self.data = data
         # initialize parameter values from JSON
-        self.ccc_initialize(call_tc=call_tc)
+        self.ccc_initialize(
+            call_tc=call_tc,
+            gfactors=gfactors,
+            weights=weights,
+            records_start_year=records_start_year,
+        )
 
-    def ccc_initialize(self, call_tc=False):
+    def ccc_initialize(
+        self,
+        call_tc=False,
+        gfactors=None,
+        weights=None,
+        records_start_year=RECORDS_START_YEAR,
+    ):
         """
         ParametersBase reads JSON file and sets attributes to self
         Next call self.compute_default_params for further initialization
 
         Args:
+            test (bool): whether to run in test mode
+            year (int): start year for simulation
             call_tc (bool): whether to use Tax-Calculator to estimate
                 marginal tax rates
+            baseline_policy (dict): individual income tax baseline
+                policy parameters, reform dict makes changes relative
+                to this baseline
+            iit_reform (dict): individual income tax reform parameters
+            data (str): data source for Tax-Calculator
+            gfactors (dict): growth factors for Tax-Calculator
+            weights (str): weights for Tax-Calculator
 
         Returns:
             None
@@ -57,11 +78,13 @@ def ccc_initialize(self, call_tc=False):
         if call_tc:
             # Find individual income tax rates from Tax-Calculator
             indiv_rates = get_rates(
-                self.baseline,
                 self.year,
                 self.baseline_policy,
                 self.iit_reform,
                 self.data,
+                gfactors,
+                weights,
+                records_start_year,
             )
             self.tau_pt = indiv_rates["tau_pt"]
             self.tau_div = indiv_rates["tau_div"]

ccc/tests/test_get_taxcalc_rates.py

@@ -6,53 +6,70 @@
 
 
 @pytest.mark.parametrize(
-    "baseline",
-    [(True), (False)],
+    "reform",
+    [(None), ({"FICA_ss_trt_employee": {2018: 0.0625}})],
     ids=["baseline", "reform"],
 )
-def test_get_calculator_cps(baseline):
+def test_get_calculator_cps(reform):
     """
     Test the get_calculator() function
     """
     calc1 = tc.get_calculator(
-        baseline,
         2019,
-        baseline_policy={"FICA_ss_trt": {2018: 0.15}},
-        reform={"FICA_ss_trt": {2018: 0.125}},
+        baseline_policy={"FICA_ss_trt_employee": {2018: 0.075}},
+        reform=reform,
     )
     assert calc1.current_year == 2019
-    if baseline:
-        assert calc1.policy_param("FICA_ss_trt") == 0.15
+    if reform is None:
+        assert calc1.policy_param("FICA_ss_trt_employee") == 0.075
     else:
-        assert calc1.policy_param("FICA_ss_trt") == 0.125
+        assert calc1.policy_param("FICA_ss_trt_employee") == 0.0625
 
 
 @pytest.mark.needs_puf
 @pytest.mark.parametrize(
-    "baseline,data",
-    [(True, "puf.csv"), (True, None), (False, None)],
-    ids=["baseline,data=PUF", "baseline,data=None", "reform,data=None"],
+    "data",
+    [("puf.csv"), (None)],
+    ids=["baseline,data=PUF", "baseline,data=None"],
 )
-def test_get_calculator(baseline, data):
+def test_get_calculator_puf(data):
     """
     Test the get_calculator() function
     """
     calc1 = tc.get_calculator(
-        baseline,
         2019,
-        baseline_policy={"FICA_ss_trt": {2018: 0.15}},
-        reform={"FICA_ss_trt": {2018: 0.125}},
+        baseline_policy={"FICA_ss_trt_employee": {2018: 0.075}},
+        reform={"FICA_ss_trt_employee": {2018: 0.0625}},
         data=data,
     )
     assert calc1.current_year == 2019
 
 
+@pytest.mark.needs_tmd
+@pytest.mark.parametrize(
+    "data",
+    [("tmd.csv")],
+    ids=["baseline,data=TMD"],
+)
+def test_get_calculator_tmd(data):
+    """
+    Test the get_calculator() function
+    """
+    calc1 = tc.get_calculator(
+        2021,
+        baseline_policy={"FICA_ss_trt_employee": {2021: 0.075}},
+        reform={"FICA_ss_trt_employee": {2022: 0.0625}},
+        data=data,
+    )
+    assert calc1.current_year == 2021
+
+
 def test_get_calculator_exception():
     """
     Test the get_calculator() function
     """
     with pytest.raises(Exception):
-        assert tc.get_calculator(True, TC_LAST_YEAR + 1)
+        assert tc.get_calculator(TC_LAST_YEAR + 1)
 
 
 def test_get_rates():
@@ -61,7 +78,6 @@ def test_get_rates():
     """
     p = Specification(year=2020)  # has default tax rates, with should equal TC
     test_dict = tc.get_rates(
-        baseline=False,
         start_year=2020,
         baseline_policy={},
         reform={},

ccc/tests/test_start_years.py

@@ -12,7 +12,7 @@ def test_tc_start_year(year):
     Test that different start years work in functions calling
     Tax-Calculator
     """
-    get_rates(True, year)
+    get_rates(year)
 
 
 @pytest.mark.parametrize(