From 2c0d1dfb86058171c07870fd313eacccbd07b8d4 Mon Sep 17 00:00:00 2001 From: Ben Watts Date: Wed, 28 Feb 2024 11:48:12 +0000 Subject: [PATCH 1/6] add error handling for backend for missing elexon data --- backend/lambda/gb_snapshot/bm/boalf.py | 7 ++++++- backend/lambda/gb_snapshot/bm/mels.py | 8 +++++++- backend/lambda/gb_snapshot/bm/pn.py | 8 +++++++- backend/lib/kilowatts-grid-stack.ts | 2 +- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/backend/lambda/gb_snapshot/bm/boalf.py b/backend/lambda/gb_snapshot/bm/boalf.py index 21c0c59..7df74ac 100644 --- a/backend/lambda/gb_snapshot/bm/boalf.py +++ b/backend/lambda/gb_snapshot/bm/boalf.py @@ -58,8 +58,13 @@ def _parse_raw_data(self, raw: RawBoalfResponse) -> pd.DataFrame: logging.info( f"parsing {len(raw.data)} pn records for {self.params.model_dump_json()}..." ) + data = [r.model_dump() for r in raw.data] + if len(data) == 0: + logging.info(f"no boalf data for {self.params}") + output = {} + return pd.DataFrame(columns=["level", "delta"], index=output.keys()) - df = pd.DataFrame([r.model_dump() for r in raw.data]) + df = pd.DataFrame(data) df = df.dropna(subset=["bmUnit"]) output = {} for bmUnit, group in df.groupby("bmUnit"): diff --git a/backend/lambda/gb_snapshot/bm/mels.py b/backend/lambda/gb_snapshot/bm/mels.py index a891080..d101ec2 100644 --- a/backend/lambda/gb_snapshot/bm/mels.py +++ b/backend/lambda/gb_snapshot/bm/mels.py @@ -43,8 +43,14 @@ def _parse_raw_data(self, raw: RawMelsResponse) -> pd.Series: logging.info( f"parsing {len(raw.data)} mels records for {self.params.model_dump_json()}..." ) + data = [r.model_dump() for r in raw.data] + if len(data) == 0: + raise Exception( + "no data returned from mels request. Likely a problem with the API." + ) + + df = pd.DataFrame(data) - df = pd.DataFrame([r.model_dump() for r in raw.data]) df = df.dropna(subset=["bmUnit"]) output = {} for bmUnit, group in df.groupby("bmUnit"): diff --git a/backend/lambda/gb_snapshot/bm/pn.py b/backend/lambda/gb_snapshot/bm/pn.py index 2aab63e..cdf7e7e 100644 --- a/backend/lambda/gb_snapshot/bm/pn.py +++ b/backend/lambda/gb_snapshot/bm/pn.py @@ -43,7 +43,13 @@ def _parse_raw_data(self, raw: RawPnResponse) -> pd.DataFrame: logging.info( f"parsing {len(raw.data)} pn records for {self.params.model_dump_json()}..." ) - df = pd.DataFrame([r.model_dump() for r in raw.data]) + + data = [r.model_dump() for r in raw.data] + if len(data) == 0: + raise Exception( + "no data returned from mels request. Likely a problem with the API." + ) + df = pd.DataFrame(data) df = df.dropna(subset=["bmUnit"]) output = {} for bmUnit, group in df.groupby("bmUnit"): diff --git a/backend/lib/kilowatts-grid-stack.ts b/backend/lib/kilowatts-grid-stack.ts index b7a8cb2..57eca24 100644 --- a/backend/lib/kilowatts-grid-stack.ts +++ b/backend/lib/kilowatts-grid-stack.ts @@ -88,7 +88,7 @@ export class KilowattsGridStack extends cdk.Stack { functionName: "gb-snapshot", code: lambda.Code.fromAsset("lambda"), handler: "gb_snapshot.handler.handler", - timeout: cdk.Duration.seconds(60), + timeout: cdk.Duration.seconds(90), memorySize: 512, environment: { BUCKET_NAME: bucket.bucketName, From 852882dfe037e02970bf4c37fd1d9e4455d02be8 Mon Sep 17 00:00:00 2001 From: Ben Watts Date: Wed, 28 Feb 2024 12:11:46 +0000 Subject: [PATCH 2/6] add stale data component --- components/gb-live/live.web.tsx | 20 +++++---- components/gb-live/stale-data-card.tsx | 56 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 components/gb-live/stale-data-card.tsx diff --git a/components/gb-live/live.web.tsx b/components/gb-live/live.web.tsx index b223a47..ce6d7b8 100644 --- a/components/gb-live/live.web.tsx +++ b/components/gb-live/live.web.tsx @@ -6,6 +6,7 @@ import { GbTotalsList } from "./bottom-sheet-tabs/totals-list/totals-list"; import { GbUnitGroupsList } from "./bottom-sheet-tabs/unit-groups-list/unit-groups-list"; import SvgMap from "./svg-map/svg-map"; import NativeAppDownloadLinks from "./native-app-download-links"; +import StaleDataCard from "./stale-data-card"; const MapTab: React.FC = () => { return ( @@ -42,14 +43,17 @@ const Live: React.FC = () => { List App - {currentTab === 0 && } - {currentTab === 1 && } - {currentTab === 2 && ( -
- -
- )} - {currentTab === 3 && } + <> + + {currentTab === 0 && } + {currentTab === 1 && } + {currentTab === 2 && ( +
+ +
+ )} + {currentTab === 3 && } + ); }; diff --git a/components/gb-live/stale-data-card.tsx b/components/gb-live/stale-data-card.tsx new file mode 100644 index 0000000..f97d0ed --- /dev/null +++ b/components/gb-live/stale-data-card.tsx @@ -0,0 +1,56 @@ +import React from "react"; +import { Linking, View } from "react-native"; +import { Button, Card, Text } from "@rneui/themed"; + +import { useGbSummaryOutputQuery } from "../../state/apis/cloudfront/api"; +import { GbSummaryOutputResponse } from "../../state/apis/cloudfront/types"; + +interface StaleDataCardProps {} + +const THRESHOLD_MINUTES = 5; + +const determineIfDataIsStale = (data: GbSummaryOutputResponse) => { + if (!data) return true; + const lastUpdated = new Date(data.dt); + const now = new Date(); + const diff = now.getTime() - lastUpdated.getTime(); + return diff > THRESHOLD_MINUTES * 60 * 1000; +}; + +const StaleDataCard: React.FC = () => { + const data = useGbSummaryOutputQuery(); + if (!data.currentData) return undefined; + const isStale = determineIfDataIsStale(data.data); + if (!isStale) return undefined; + return ( + + + Data Error + + {`Data shown is from ${new Date(Date.parse(data.data.dt)).toLocaleString()} and is stale/out-of-date. There may be an outage affecting Elexon/BMRS data, our data pipeline, or your internet connection. `} + + + + + + + + + + + ); +}; +export default StaleDataCard; From 7227665a61fc29eed601e68f66991405edec61a5 Mon Sep 17 00:00:00 2001 From: Ben Watts Date: Wed, 28 Feb 2024 12:28:50 +0000 Subject: [PATCH 3/6] add stale data card --- .../gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx | 4 +++- .../bottom-sheet-tabs/unit-groups-list/unit-groups-list.tsx | 2 ++ components/gb-live/live.tsx | 1 + components/gb-live/stale-data-card.tsx | 5 +++-- state/apis/cloudfront/api.ts | 2 +- 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/components/gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx b/components/gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx index 0356fc4..66a17a7 100644 --- a/components/gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx +++ b/components/gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx @@ -1,5 +1,5 @@ import React from "react"; -import { FlatList, StyleSheet, View } from "react-native"; +import { FlatList, Platform, StyleSheet, View } from "react-native"; import { Button, Card, Icon, Text } from "@rneui/themed"; import { useGbSummaryOutputQuery } from "../../../../state/apis/cloudfront/api"; @@ -9,6 +9,7 @@ import { calculateCycleSeconds } from "../../../../state/utils"; import { ErrorDataRetryCard } from "../../error-data-retry-card"; +import StaleDataCard from "../../stale-data-card"; import { GbLiveListItem } from "../live-list-item/live-list-item"; import { GbBalancingTotals } from "./balancing-totals/balancing-totals"; @@ -33,6 +34,7 @@ export const GbTotalsList: React.FC = () => { data={data && data.totals} refreshing={isLoading} onRefresh={() => refetch()} + ListHeaderComponent={Platform.OS !== "web" && StaleDataCard} renderItem={({ item }) => ( { code={item.code} /> )} + ListHeaderComponent={Platform.OS !== "web" && StaleDataCard} ListFooterComponent={ Platform.OS === "web" ? ( { - if (!data) return true; + return true; + if (!data) return false; const lastUpdated = new Date(data.dt); const now = new Date(); const diff = now.getTime() - lastUpdated.getTime(); diff --git a/state/apis/cloudfront/api.ts b/state/apis/cloudfront/api.ts index 5f15435..9c4a5ca 100644 --- a/state/apis/cloudfront/api.ts +++ b/state/apis/cloudfront/api.ts @@ -8,7 +8,7 @@ import { GbSummaryOutputResponse } from "./types"; export const kilowattsCloudfront = createApi({ reducerPath: "kilowattsCloudfront", baseQuery: fetchBaseQuery({ - baseUrl: `https://${EXPO_PUBLIC_CDN_DOMAIN_NAME}` + baseUrl: `https://gridcdn.kilowatts.io` }), endpoints: (builder) => ({ gbSummaryOutput: builder.query({ From 764d14cf1b52466b96b1dccce08a4aa24d894dfa Mon Sep 17 00:00:00 2001 From: Ben Watts Date: Wed, 28 Feb 2024 12:30:53 +0000 Subject: [PATCH 4/6] rollback --- state/apis/cloudfront/api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/apis/cloudfront/api.ts b/state/apis/cloudfront/api.ts index 9c4a5ca..5f15435 100644 --- a/state/apis/cloudfront/api.ts +++ b/state/apis/cloudfront/api.ts @@ -8,7 +8,7 @@ import { GbSummaryOutputResponse } from "./types"; export const kilowattsCloudfront = createApi({ reducerPath: "kilowattsCloudfront", baseQuery: fetchBaseQuery({ - baseUrl: `https://gridcdn.kilowatts.io` + baseUrl: `https://${EXPO_PUBLIC_CDN_DOMAIN_NAME}` }), endpoints: (builder) => ({ gbSummaryOutput: builder.query({ From 1f469d9e04e60264ed38b2c475ae3d342f800839 Mon Sep 17 00:00:00 2001 From: Ben Watts Date: Wed, 28 Feb 2024 12:31:42 +0000 Subject: [PATCH 5/6] remove automatic stale --- components/gb-live/stale-data-card.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/components/gb-live/stale-data-card.tsx b/components/gb-live/stale-data-card.tsx index bb8f242..b8be683 100644 --- a/components/gb-live/stale-data-card.tsx +++ b/components/gb-live/stale-data-card.tsx @@ -10,7 +10,6 @@ interface StaleDataCardProps {} const THRESHOLD_MINUTES = 10; const determineIfDataIsStale = (data: GbSummaryOutputResponse) => { - return true; if (!data) return false; const lastUpdated = new Date(data.dt); const now = new Date(); From f1f947125a94a89156d01f562e6968fcbb604c7d Mon Sep 17 00:00:00 2001 From: Ben Watts Date: Wed, 28 Feb 2024 12:33:54 +0000 Subject: [PATCH 6/6] update polling intervals --- .../totals-list/balancing-totals/balancing-totals.tsx | 2 +- .../gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx | 5 ++--- .../bottom-sheet-tabs/unit-groups-list/unit-groups-list.tsx | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/components/gb-live/bottom-sheet-tabs/totals-list/balancing-totals/balancing-totals.tsx b/components/gb-live/bottom-sheet-tabs/totals-list/balancing-totals/balancing-totals.tsx index ea82b70..977c4d3 100644 --- a/components/gb-live/bottom-sheet-tabs/totals-list/balancing-totals/balancing-totals.tsx +++ b/components/gb-live/bottom-sheet-tabs/totals-list/balancing-totals/balancing-totals.tsx @@ -8,7 +8,7 @@ import { GbLiveListItemBalancingTotal } from "../../live-list-item/live-list-ite export const GbBalancingTotals = () => { const { data } = useGbSummaryOutputQuery(undefined, { - pollingInterval: 1000 * 15 + pollingInterval: 1000 * 60 }); return ( diff --git a/components/gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx b/components/gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx index 66a17a7..8370367 100644 --- a/components/gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx +++ b/components/gb-live/bottom-sheet-tabs/totals-list/totals-list.tsx @@ -1,6 +1,5 @@ import React from "react"; -import { FlatList, Platform, StyleSheet, View } from "react-native"; -import { Button, Card, Icon, Text } from "@rneui/themed"; +import { FlatList, Platform } from "react-native"; import { useGbSummaryOutputQuery } from "../../../../state/apis/cloudfront/api"; import { @@ -20,7 +19,7 @@ export const GbTotalsList: React.FC = () => { const { data, isLoading, refetch, isError } = useGbSummaryOutputQuery( undefined, { - pollingInterval: 1000 * 15, + pollingInterval: 1000 * 60, refetchOnReconnect: true } ); diff --git a/components/gb-live/bottom-sheet-tabs/unit-groups-list/unit-groups-list.tsx b/components/gb-live/bottom-sheet-tabs/unit-groups-list/unit-groups-list.tsx index a9dbad1..b29f368 100644 --- a/components/gb-live/bottom-sheet-tabs/unit-groups-list/unit-groups-list.tsx +++ b/components/gb-live/bottom-sheet-tabs/unit-groups-list/unit-groups-list.tsx @@ -24,7 +24,7 @@ export const GbUnitGroupsList: React.FC = () => { const { data, isLoading, refetch, isError } = useGbSummaryOutputQuery( undefined, { - pollingInterval: 1000 * 15, + pollingInterval: 1000 * 60, refetchOnReconnect: true } );