From 8748024c7832720156ef34905e93a850cbb256e6 Mon Sep 17 00:00:00 2001 From: ulixius9 Date: Tue, 23 Jan 2024 14:22:14 +0530 Subject: [PATCH] Minor: Bump Version & Fix Merge Changes --- ingestion/Dockerfile | 2 +- ingestion/operators/docker/Dockerfile | 2 +- ingestion/setup.py | 2 +- .../source/database/databricks/client.py | 52 +++++++++++++++++++ openmetadata-airflow-apis/setup.py | 2 +- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index 755e73acb27c..251e650a9ca7 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -81,7 +81,7 @@ ARG INGESTION_DEPENDENCY="all" ENV PIP_NO_CACHE_DIR=1 # Make pip silent ENV PIP_QUIET=1 -ARG RI_VERSION="1.2.4.2" +ARG RI_VERSION="1.2.4.4" RUN pip install --upgrade pip RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.6.3/constraints-3.9.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index 71ec51393ea9..102b3745dcb0 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -87,7 +87,7 @@ ENV PIP_QUIET=1 RUN pip install --upgrade pip ARG INGESTION_DEPENDENCY="all" -ARG RI_VERSION="1.2.4.2" +ARG RI_VERSION="1.2.4.4" RUN pip install --upgrade pip RUN pip install "openmetadata-ingestion[airflow]~=${RI_VERSION}" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" diff --git a/ingestion/setup.py b/ingestion/setup.py index c3de6a99acd3..b1e09a36d1a0 100644 --- a/ingestion/setup.py +++ b/ingestion/setup.py @@ -311,7 +311,7 @@ def get_long_description(): build_options = {"includes": ["_cffi_backend"]} setup( name="openmetadata-ingestion", - version="1.2.4.3", + version="1.2.4.4", url="https://open-metadata.org/", author="OpenMetadata Committers", license="Apache License 2.0", diff --git a/ingestion/src/metadata/ingestion/source/database/databricks/client.py b/ingestion/src/metadata/ingestion/source/database/databricks/client.py index 8f86634e10e1..b2febbff64a0 100644 --- a/ingestion/src/metadata/ingestion/source/database/databricks/client.py +++ b/ingestion/src/metadata/ingestion/source/database/databricks/client.py @@ -225,3 +225,55 @@ def get_job_runs(self, job_id) -> List[dict]: except Exception as exc: logger.debug(traceback.format_exc()) logger.error(exc) + + def get_table_lineage(self, table_name: str) -> LineageTableStreams: + """ + Method returns table lineage details + """ + try: + data = { + "table_name": table_name, + } + + response = self.client.get( + f"{self.base_url}{TABLE_LINEAGE_PATH}", + headers=self.headers, + data=json.dumps(data), + timeout=API_TIMEOUT, + ).json() + if response: + return LineageTableStreams(**response) + + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.error(exc) + + return LineageTableStreams() + + def get_column_lineage( + self, table_name: str, column_name: str + ) -> LineageColumnStreams: + """ + Method returns table lineage details + """ + try: + data = { + "table_name": table_name, + "column_name": column_name, + } + + response = self.client.get( + f"{self.base_url}{COLUMN_LINEAGE_PATH}", + headers=self.headers, + data=json.dumps(data), + timeout=API_TIMEOUT, + ).json() + + if response: + return LineageColumnStreams(**response) + + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.error(exc) + + return LineageColumnStreams() diff --git a/openmetadata-airflow-apis/setup.py b/openmetadata-airflow-apis/setup.py index 426454ddaaab..55b0290e35f8 100644 --- a/openmetadata-airflow-apis/setup.py +++ b/openmetadata-airflow-apis/setup.py @@ -69,7 +69,7 @@ def get_long_description(): packages=find_packages(include=[f"{PLUGIN_NAME}.*", PLUGIN_NAME]), include_package_data=True, package_data={PLUGIN_NAME: get_package_data()}, - version="1.2.4.2", + version="1.2.4.4", url="https://open-metadata.org/", author="OpenMetadata Committers", license="Apache License 2.0",