Merge pull request #114 from dbt-labs/release-1.0

Preparing for release 1.0.0

.pre-commit-config.yaml

@@ -1,10 +1,10 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.4.9
+    rev: v0.8.2
     hooks:
       # Run the linter.
       - id: ruff
-        args: [ --select, I, B, --fix ]
+        args: [ --select, I, --select, B, --fix ]
       # Run the formatter.
       - id: ruff-format

pyproject.toml

@@ -20,7 +20,7 @@ dependencies = [
     "importlib-metadata<7,>=6.0",
 ]
 name = "dbt-jobs-as-code"
-version = "0.11.1"
+version = "1.0.0"
 description = "A CLI to allow defining dbt Cloud jobs as code"
 readme = "README.md"
 keywords = [
@@ -38,6 +38,7 @@ dbt-jobs-as-code = "dbt_jobs_as_code.main:cli"
 dev = [
     "coverage<8.0.0,>=7.6.3",
     "jsonschema<5.0.0,>=4.17.3",
+    "pytest-mock>=3.14.0",
     "pytest<8.0.0,>=7.2.0",
     "pytest-beartype<1.0.0,>=0.0.2",
     "pytest-cov<6.0.0,>=5.0.0",

src/dbt_jobs_as_code/cloud_yaml_mapping/change_set.py

@@ -4,13 +4,14 @@
 
 from beartype import BeartypeConf, BeartypeStrategy, beartype
 from beartype.typing import Callable, List
+from loguru import logger
+from pydantic import BaseModel, RootModel
+from rich.table import Table
+
 from dbt_jobs_as_code.client import DBTCloud, DBTCloudException
 from dbt_jobs_as_code.loader.load import load_job_configuration
 from dbt_jobs_as_code.schemas import check_env_var_same, check_job_mapping_same
 from dbt_jobs_as_code.schemas.job import JobDefinition
-from loguru import logger
-from pydantic import BaseModel, RootModel
-from rich.table import Table
 
 # Dynamically create a new @nobeartype decorator disabling type-checking.
 nobeartype = beartype(conf=BeartypeConf(strategy=BeartypeStrategy.O0))

src/dbt_jobs_as_code/importer/__init__.py

@@ -0,0 +1,45 @@
+from beartype.typing import List, Optional, TextIO
+from loguru import logger
+
+from dbt_jobs_as_code.client import DBTCloud
+from dbt_jobs_as_code.loader.load import load_job_configuration
+from dbt_jobs_as_code.schemas.job import JobDefinition
+
+
+def get_account_id(config_file: Optional[TextIO], account_id: Optional[int]) -> int:
+    """Get account ID from either config file or direct input"""
+    if account_id:
+        return account_id
+    elif config_file:
+        defined_jobs = load_job_configuration(config_file, None).jobs.values()
+        return list(defined_jobs)[0].account_id
+    else:
+        raise ValueError("Either config or account_id must be provided")
+
+
+def check_job_fields(dbt_cloud: DBTCloud, job_ids: List[int]) -> None:
+    """Check if there are new fields in job model"""
+    if not job_ids:
+        logger.error("We need to provide some job_id to test the import")
+        return
+
+    logger.info("Checking if there are new fields for jobs")
+    dbt_cloud.get_job_missing_fields(job_id=job_ids[0])
+
+
+def fetch_jobs(
+    dbt_cloud: DBTCloud, job_ids: List[int], project_ids: List[int], environment_ids: List[int]
+) -> List[JobDefinition]:
+    """Fetch jobs from dbt Cloud based on provided filters"""
+    logger.info("Getting the jobs definition from dbt Cloud")
+
+    if job_ids and not (project_ids or environment_ids):
+        # Get jobs one by one if only job_ids provided
+        cloud_jobs_can_have_none = [dbt_cloud.get_job(job_id=id) for id in job_ids]
+        return [job for job in cloud_jobs_can_have_none if job is not None]
+
+    # Get all jobs and filter
+    cloud_jobs = dbt_cloud.get_jobs(project_ids=project_ids, environment_ids=environment_ids)
+    if job_ids:
+        cloud_jobs = [job for job in cloud_jobs if job.id in job_ids]
+    return cloud_jobs

src/dbt_jobs_as_code/loader/load.py

@@ -67,7 +67,7 @@ def _load_yaml_with_template(config_file: TextIO, vars_file: TextIO) -> dict:
         config_string_rendered = template.render(template_vars_values)
     except UndefinedError as e:
         print(f"Error: {e}")  # This will raise an error
-        raise LoadingJobsYAMLError(f"Some variables didn't have a value: {e.message}.")
+        raise LoadingJobsYAMLError(f"Some variables didn't have a value: {e.message}.") from e
 
     return yaml.safe_load(config_string_rendered)
 

src/dbt_jobs_as_code/main.py

@@ -11,6 +11,7 @@
 from dbt_jobs_as_code.cloud_yaml_mapping.change_set import build_change_set
 from dbt_jobs_as_code.cloud_yaml_mapping.validate_link import LinkableCheck, can_be_linked
 from dbt_jobs_as_code.exporter.export import export_jobs_yml
+from dbt_jobs_as_code.importer import check_job_fields, fetch_jobs, get_account_id
 from dbt_jobs_as_code.loader.load import load_job_configuration
 from dbt_jobs_as_code.schemas.config import generate_config_schema
 
@@ -308,22 +309,10 @@ def import_jobs(
     """
 
     # we get the account id either from a parameter (e.g if the config file doesn't exist) or from the config file
-    if account_id:
-        cloud_account_id = account_id
-    elif config:
-        defined_jobs = load_job_configuration(config, None).jobs.values()
-        cloud_account_id = list(defined_jobs)[0].account_id
-    else:
-        raise click.BadParameter("Either --config or --account-id must be provided")
-
-    cloud_project_ids = []
-    cloud_environment_ids = []
-
-    if project_id:
-        cloud_project_ids = project_id
-
-    if environment_id:
-        cloud_environment_ids = environment_id
+    try:
+        cloud_account_id = get_account_id(config, account_id)
+    except ValueError as e:
+        raise click.BadParameter(str(e)) from e
 
     dbt_cloud = DBTCloud(
         account_id=cloud_account_id,
@@ -332,33 +321,15 @@ def import_jobs(
         disable_ssl_verification=disable_ssl_verification,
     )
 
-    # this is a special case to check if there are new fields in the job model
     if check_missing_fields:
-        if not job_id:
-            logger.error("We need to provide some job_id to test the import")
-        else:
-            logger.info(f"Checking if there are new fields for jobs")
-            # retrieve the job and raise errors if there are new fields
-            dbt_cloud.get_job_missing_fields(job_id=job_id[0])
+        check_job_fields(dbt_cloud, list(job_id))
         return
 
-    # we want to avoid querying all jobs if it's not needed
-    # if we don't provide a filter for project/env but provide a list of job ids, we get the jobs one by one
-    elif job_id and not (cloud_project_ids or cloud_environment_ids):
-        logger.info(f"Getting the jobs definition from dbt Cloud")
-        cloud_jobs_can_have_none = [dbt_cloud.get_job(job_id=id) for id in job_id]
-        cloud_jobs = [job for job in cloud_jobs_can_have_none if job is not None]
-    # otherwise, we get all the jobs and filter the list
-    else:
-        logger.info(f"Getting the jobs definition from dbt Cloud")
-        cloud_jobs = dbt_cloud.get_jobs(
-            project_ids=cloud_project_ids, environment_ids=cloud_environment_ids
-        )
-        if job_id:
-            cloud_jobs = [job for job in cloud_jobs if job.id in job_id]
+    cloud_jobs = fetch_jobs(dbt_cloud, list(job_id), list(project_id), list(environment_id))
 
+    # Handle env vars
     for cloud_job in cloud_jobs:
-        logger.info(f"Getting en vars_yml overwrites for the job {cloud_job.id}:{cloud_job.name}")
+        logger.info(f"Getting env vars overwrites for job {cloud_job.id}:{cloud_job.name}")
         env_vars = dbt_cloud.get_env_vars(
             project_id=cloud_job.project_id,
             job_id=cloud_job.id,  # type: ignore # in that case, we have an ID as we are importing
@@ -367,7 +338,7 @@ def import_jobs(
             if env_var.value:
                 cloud_job.custom_environment_variables.append(env_var)
 
-    logger.success(f"YML file for the current dbt Cloud jobs")
+    logger.success("YML file for the current dbt Cloud jobs")
     export_jobs_yml(cloud_jobs, include_linked_id)
 
 

src/dbt_jobs_as_code/schemas/__init__.py

@@ -1,3 +1,5 @@
+import json
+
 from beartype.typing import Any, Optional, Tuple
 from deepdiff import DeepDiff
 from loguru import logger
@@ -34,13 +36,15 @@ def check_job_mapping_same(source_job: JobDefinition, dest_job: JobDefinition) -
     source_job_dict = _job_to_dict(source_job)
     dest_job_dict = _job_to_dict(dest_job)
 
-    diffs = _get_mismatched_dict_entries(source_job_dict, dest_job_dict)
+    diffs = _get_mismatched_dict_entries(dest_job_dict, source_job_dict)
 
     if len(diffs) == 0:
         logger.success(f"✅ Job {source_job.identifier} is identical")
         return True
     else:
-        logger.info(f"❌ Job {source_job.identifier} is different - Diff: {diffs}")
+        logger.info(
+            f"❌ Job {source_job.identifier} is different - Diff:\n{json.dumps(diffs, indent=2)}"
+        )
         return False
 
 

src/dbt_jobs_as_code/schemas/config.py

@@ -13,7 +13,7 @@ class Config(BaseModel):
 
     def __init__(self, **data: Any):
         # Check for instances where account_id is missing from a job, and add it from the config data.
-        for identifier, job in data.get("jobs", dict()).items():
+        for job in data.get("jobs", dict()).values():
             if "account_id" not in job or job["account_id"] is None:
                 job["account_id"] = data["account_id"]
 

src/dbt_jobs_as_code/schemas/custom_environment_variable.py

@@ -6,7 +6,7 @@
 class CustomEnvironmentVariable(BaseModel):
     name: str
     type: Literal["project", "environment", "job", "user"] = "job"
-    value: Optional[str] = None
+    value: Optional[str] = Field(default=None)
     display_value: Optional[str] = None
     job_definition_id: Optional[int] = None
 

tests/importer/test_importer.py

@@ -0,0 +1,61 @@
+from unittest.mock import Mock
+
+import pytest
+
+from dbt_jobs_as_code.importer import fetch_jobs, get_account_id
+from dbt_jobs_as_code.schemas.job import JobDefinition
+
+
+def test_get_account_id():
+    # Test account ID from direct input
+    assert get_account_id(None, 123) == 123
+
+    # Test missing both inputs
+    with pytest.raises(ValueError):
+        get_account_id(None, None)
+
+
+def test_fetch_jobs():
+    mock_dbt = Mock()
+
+    # Mock job objects
+    mock_job1 = JobDefinition(
+        id=1,
+        name="Job 1",
+        project_id=100,
+        environment_id=200,
+        account_id=300,
+        settings={},
+        run_generate_sources=False,
+        execute_steps=[],
+        generate_docs=False,
+        schedule={"cron": "0 14 * * 0,1,2,3,4,5,6"},
+        triggers={},
+    )
+    mock_job2 = JobDefinition(
+        id=2,
+        name="Job 2",
+        project_id=100,
+        environment_id=200,
+        account_id=300,
+        settings={},
+        run_generate_sources=False,
+        execute_steps=[],
+        generate_docs=False,
+        schedule={"cron": "0 14 * * 0,1,2,3,4,5,6"},
+        triggers={},
+    )
+
+    # Set return values for mocks
+    mock_dbt.get_job.side_effect = [mock_job1, mock_job2]
+    mock_dbt.get_jobs.return_value = [mock_job1, mock_job2]
+
+    # Test fetch with only job IDs
+    jobs = fetch_jobs(mock_dbt, [1, 2], [], [])
+    assert mock_dbt.get_job.call_count == 2
+    assert len(jobs) == 2
+
+    # Test fetch with project IDs
+    jobs = fetch_jobs(mock_dbt, [1], [100], [])
+    mock_dbt.get_jobs.assert_called_with(project_ids=[100], environment_ids=[])
+    assert len(jobs) == 1

tests/schemas/test_custom_environment_variables.py

@@ -0,0 +1,27 @@
+import pytest
+from pydantic import ValidationError
+
+from dbt_jobs_as_code.schemas.custom_environment_variable import (
+    CustomEnvironmentVariable,
+    CustomEnvironmentVariablePayload,
+)
+
+
+def test_custom_env_var_validation():
+    """Test that environment variable validation works correctly"""
+
+    # Valid cases
+    valid_var = CustomEnvironmentVariable(name="DBT_TEST_VAR", value="test_value")
+    assert valid_var.name == "DBT_TEST_VAR"
+    assert valid_var.value == "test_value"
+    assert valid_var.type == "job"
+
+    # Test invalid prefix
+    with pytest.raises(ValidationError) as exc:
+        CustomEnvironmentVariable(name="TEST_VAR", value="test")
+    assert "Key must have `DBT_` prefix" in str(exc.value)
+
+    # Test lowercase name
+    with pytest.raises(ValidationError) as exc:
+        CustomEnvironmentVariable(name="DBT_test_var", value="test")
+    assert "Key name must be SCREAMING_SNAKE_CASE" in str(exc.value)