Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix activities stream state #7

Merged
merged 10 commits into from
Dec 21, 2023
Merged

Fix activities stream state #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0e7088f
ads stream ready for prod
NahidOulmi Dec 15, 2023
fcabc08
add activities sample state
NahidOulmi Dec 18, 2023
558e5db
merge with master
NahidOulmi Dec 18, 2023
be5776d
try fix in images stream read records method
NahidOulmi Dec 18, 2023
f747a60
wip
NahidOulmi Dec 19, 2023
eb69bd2
ready
NahidOulmi Dec 19, 2023
f483839
ready for PR
NahidOulmi Dec 19, 2023
3d648f2
fix insights stream state by using stream slice & herit get_updated_s…
NahidOulmi Dec 20, 2023
335e2c4
remove get_updated_state herited method from Activities stream
NahidOulmi Dec 21, 2023
dec671f
change configured_catalog.json
NahidOulmi Dec 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
224 changes: 0 additions & 224 deletions ...tegrations/connectors/source-facebook-marketing/integration_tests/configured_catalog.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,110 +1,5 @@
{
"streams": [
{
"stream": {
"name": "ad_account",
"json_schema": {},
"supported_sync_modes": [
"full_refresh"
]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads_insights",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "images",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "images",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ad_account",
"json_schema": {},
"supported_sync_modes": [
"full_refresh"
]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ad_sets",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ad_creatives",
"json_schema": {},
"supported_sync_modes": [
"full_refresh"
]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads_insights",
Expand All @@ -117,125 +12,6 @@
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads_insights_age_and_gender",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads_insights_country",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads_insights_region",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads_insights_dma",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads_insights_platform_and_device",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "ads_insights_action_type",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "campaigns",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "custom_conversions",
"json_schema": {},
"supported_sync_modes": [
"full_refresh"
]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "images",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "videos",
"json_schema": {},
"supported_sync_modes": [
"full_refresh",
"incremental"
]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
},
{
"stream": {
"name": "activities",
Expand Down
80 changes: 65 additions & 15 deletions ...yte-integrations/connectors/source-facebook-marketing/integration_tests/sample_state.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,39 @@
[
{
{
"type": "STREAM",
"stream": {
"stream_state": {
"1101472513921186": {
"slices": [
"2023-12-07"
],
"updated_time": "2023-12-07"
"date_start": "2023-12-19",
"time_increment": 1
},
"1107987959844881": {
"slices": [
"2023-12-07"
],
"date_start": "2023-12-19",
"time_increment": 1
}
},
"stream_descriptor": {
"name": "ads_insights"
}
}
},
{
"type": "STREAM",
"stream": {
"stream_state": {
"646247609923744": {
"updated_time": "2023-12-15 19:04:24.899176+00",
"include_deleted": 1
},
"1447352185602743": {
"updated_time": "2023-12-15 19:04:24.899176+00",
"include_deleted": 1
}
},
"stream_descriptor": {
Expand All @@ -19,23 +45,47 @@
"type": "STREAM",
"stream": {
"stream_state": {
"1101472513921186": {
"slices": [
"2023-12-07"
],
"updated_time": "2023-12-07",
"time_increment": 1
"646247609923744": {
"event_time": "2023-12-18",
"include_deleted": 0
},
"1107987959844881": {
"slices": [
"2023-12-07"
],
"updated_time": "2023-12-07",
"time_increment": 1
"1447352185602743": {
"event_time": "2023-12-18",
"include_deleted": 0
}
},
"stream_descriptor": {
"name": "ads_insights"
"name": "activities"
}
}
},
{
"type": "STREAM",
"stream": {
"stream_state": {
"646247609923744": {
"updated_time": "2023-12-15 19:04:24.899176+00",
"include_deleted": false
},
"1447352185602743": {
"updated_time": "2023-12-15 19:04:24.899176+00",
"include_deleted": false
}
},
"stream_descriptor": {
"name": "ads"
}
}
},
{
"type": "STREAM",
"stream": {
"stream_state": {
"date_start": "2023-10-19T00:00:00Z",
"include_deleted": true
},
"stream_descriptor": {
"name": "ads_insights_age_and_gender"
}
}
},
Expand Down
Loading
Loading