Skip to content

Commit

Permalink
Adds extra guards for plugin imports
Browse files Browse the repository at this point in the history
We should probably just blanket catch all errors -- but
not doing that right now. Hamilton fails to be imported
in a spark serverless environment because there is some
interaction with DLT that databricks doesn't like.

So catching more exceptions -- and shoe horning it into
an import error.
  • Loading branch information
skrawcz committed Jul 2, 2024
1 parent 103902d commit 929e4fd
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 6 deletions.
2 changes: 2 additions & 0 deletions hamilton/function_modifiers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
logger.debug(f"Did not load {plugin_module} extension because {str(e)}.")
except ModuleNotFoundError as e:
logger.debug(f"Did not load {plugin_module} extension because {e.msg}.")
except ImportError as e:
logger.debug(f"Did not load {plugin_module} extension because {str(e)}.")
registry.INITIALIZED = True


Expand Down
21 changes: 15 additions & 6 deletions hamilton/plugins/dlt_extensions.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,22 @@
import dataclasses
from typing import Any, Collection, Dict, Iterable, Literal, Optional, Sequence, Tuple, Type

import dlt
import pandas as pd
from dlt.common.destination.capabilities import TLoaderFileFormat
from dlt.common.schema import Schema, TColumnSchema
try:
import dlt
from dlt.common.destination.capabilities import TLoaderFileFormat
from dlt.common.schema import Schema, TColumnSchema

# importing TDestinationReferenceArg fails if Destination isn't imported
from dlt.extract.resource import DltResource

# importing TDestinationReferenceArg fails if Destination isn't imported
from dlt.extract.resource import DltResource
except ImportError as e:
# raise import error first
raise ImportError(f"Failed to import the DLT library. {e}")
except Exception as e:
# raise import error with custom message
raise ImportError("Failed to import the DLT library.") from e

import pandas as pd

from hamilton import registry
from hamilton.io import utils
Expand Down

0 comments on commit 929e4fd

Please sign in to comment.