From 86d19f111d1c42ae6990170ca7fe799987bb5491 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 31 Dec 2024 13:34:03 -0500 Subject: [PATCH 01/12] update to v2 --- components/opensea/opensea.app.mjs | 51 +++++++--- components/opensea/package.json | 7 +- .../new-collection-events.mjs | 93 ++++++++++--------- 3 files changed, 90 insertions(+), 61 deletions(-) diff --git a/components/opensea/opensea.app.mjs b/components/opensea/opensea.app.mjs index 5c0b65d7b062c..e2139c4faf764 100644 --- a/components/opensea/opensea.app.mjs +++ b/components/opensea/opensea.app.mjs @@ -5,23 +5,48 @@ export default { app: "opensea", propDefinitions: {}, methods: { - async retrieveEvents({ - $, contract, eventType, occurredAfter, cursor, + _baseUrl() { + return "https://api.opensea.io/api/v2"; + }, + _makeRequest({ + $ = this, + path, + ...otherOpts }) { - const apiKey = this.$auth.api_key; - return axios($ ?? this, { - url: "https://api.opensea.io/api/v1/events", - params: { - only_opensea: false, - asset_contract_address: contract, - event_type: eventType, - occurred_after: occurredAfter, - cursor, - }, + return axios($, { + url: `${this._baseUrl()}${path}`, headers: { - "X-API-KEY": apiKey, + "X-API-KEY": this.$auth.api_key, }, + ...otherOpts, }); }, + retrieveEvents({ + collectionSlug, ...opts + }) { + return this._makeRequest({ + path: `/listings/collection/${collectionSlug}/all`, + ...opts, + }); + }, + async *paginate({ + fn, + args = {}, + resourceKey, + }) { + let total = 0; + do { + const response = await fn(args); + const items = response[resourceKey]; + for (const item of items) { + yield item; + } + total = items?.length; + args.params = { + ...args?.params, + next: response?.next, + }; + } while (total && args.params?.next); + }, }, }; diff --git a/components/opensea/package.json b/components/opensea/package.json index e4385afa2338d..005f26b5aacfc 100644 --- a/components/opensea/package.json +++ b/components/opensea/package.json @@ -1,21 +1,18 @@ { "name": "@pipedream/opensea", - "version": "0.0.3", + "version": "0.0.4", "description": "Pipedream Opensea Components", "main": "opensea.app.mjs", "keywords": [ "pipedream", "opensea" ], - "files": [ - "dist" - ], "homepage": "https://pipedream.com/apps/opensea", "author": "Pipedream (https://pipedream.com/)", "publishConfig": { "access": "public" }, "dependencies": { - "@pipedream/platform": "^1.2.0" + "@pipedream/platform": "^3.0.3" } } diff --git a/components/opensea/sources/new-collection-events/new-collection-events.mjs b/components/opensea/sources/new-collection-events/new-collection-events.mjs index dde3a4bb3bdab..7abfc26c8a1e8 100644 --- a/components/opensea/sources/new-collection-events/new-collection-events.mjs +++ b/components/opensea/sources/new-collection-events/new-collection-events.mjs @@ -2,12 +2,11 @@ import opensea from "../../opensea.app.mjs"; import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; export default { - name: "New Collection Events", - version: "0.0.3", key: "opensea-new-collection-events", - description: - "Emit new filtered events. [See docs](https://docs.opensea.io/reference/retrieving-asset-events)", - dedupe: "greatest", + name: "New Collection Events", + description: "Emit new listings for a collection. [See the documentation](https://docs.opensea.io/reference/get_all_listings_on_collection_v2)", + version: "0.0.4", + dedupe: "unique", type: "source", props: { opensea, @@ -18,54 +17,62 @@ export default { intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL, }, }, - contractAddress: { + collectionSlug: { type: "string", - label: "Contract Address", - description: "Collection contract address", - }, - eventType: { - type: "string", - options: [ - "sales", - "listings", - ], - label: "Event Type", - description: "OpenSea event type", + label: "Collection Slug", + description: "Unique string to identify a collection on OpenSea. This can be found by visiting the collection on the OpenSea website and noting the last path parameter.", }, }, methods: { getLastTimestamp() { - return this.db.get("lastTimestamp"); + return this.db.get("lastTimestamp") || 0; }, setLastTimestamp(ts) { this.db.set("lastTimestamp", ts); }, - }, - async run() { - const eventType = this.eventType === "sales" - ? "successful" - : "created"; - const lastTimestamp = this.getLastTimestamp(); - let cursor = null; - do { - const resp = await this.opensea.retrieveEvents({ - contract: this.contractAddress, - eventType, - occurredAfter: lastTimestamp, - cursor, - }); - resp.asset_events.forEach((event) => { - this.$emit(event, { - id: event.id, - summary: `${event.asset.name} ${this.eventType} event`, - ts: +new Date(event.created_date), - }); + generateMeta(item) { + return { + id: item.order_hash, + summary: `New ${item.type} ${item.chain} Listing`, + ts: item.protocol_data.parameters.startTime, + }; + }, + async processEvent(max) { + const lastTimestamp = this.getLastTimestamp(); + + const results = this.opensea.paginate({ + fn: this.opensea.retrieveEvents, + args: { + collectionSlug: this.collectionSlug, + }, + resourceKey: "listings", }); - if (!cursor && resp.asset_events.length > 0) { - const ts = Math.floor(new Date(resp.asset_events[0].created_date).getTime() / 1000); - this.setLastTimestamp(ts); + + let items = []; + for await (const result of results) { + const ts = result.protocol_data.parameters.startTime; + if (ts >= lastTimestamp) { + items.push(result); + } } - cursor = resp.next; - } while (lastTimestamp && cursor); + + if (max) { + items = items.slice(-1 * max); + } + this.setLastTimestamp(items[items.length - 1].protocol_data.parameters.startTime); + + items.forEach((item) => { + const meta = this.generateMeta(item); + this.$emit(item, meta); + }); + }, + }, + hooks: { + async deploy() { + await this.processEvent(25); + }, + }, + async run() { + await this.processEvent(); }, }; From 619d0ec143152fd76308c0171a04298f8fb7c6d8 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 31 Dec 2024 13:37:07 -0500 Subject: [PATCH 02/12] pnpm-lock.yaml --- pnpm-lock.yaml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c607001294094..e2541dffc9f1f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -94,8 +94,8 @@ importers: specifier: ^12.3.4 version: 12.5.0(enquirer@2.4.1) pnpm: - specifier: 9.14.3 - version: 9.14.3 + specifier: 9.14.2 + version: 9.14.2 putout: specifier: '>=36' version: 36.13.1(eslint@8.57.1)(typescript@5.6.3) @@ -7196,8 +7196,8 @@ importers: components/opensea: dependencies: '@pipedream/platform': - specifier: ^1.2.0 - version: 1.6.6 + specifier: ^3.0.3 + version: 3.0.3 components/openweather_api: dependencies: @@ -12432,7 +12432,7 @@ importers: version: 3.1.7 ts-jest: specifier: ^29.2.5 - version: 29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2) + version: 29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2) typescript: specifier: ^5.6 version: 5.7.2 @@ -23118,8 +23118,8 @@ packages: resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} engines: {node: '>=4'} - pnpm@9.14.3: - resolution: {integrity: sha512-wPU+6ZR37ZabgrKJrQEaXRa/FiPJV+fynqvo0MALV0wpuMf1T2xn7nEMc/KFyBVNB85EtG/iwO60dqkEQbrDcQ==} + pnpm@9.14.2: + resolution: {integrity: sha512-biuvd9Brk2IpQVLIUcTyeO3jerHro6Vf2jF6SheyCfTbuXP7JQp3q8Rjo0H8sfF/F8+iQJHE6zGc2g2bhCeDhw==} engines: {node: '>=18.12'} hasBin: true @@ -24579,22 +24579,22 @@ packages: superagent@3.8.1: resolution: {integrity: sha512-VMBFLYgFuRdfeNQSMLbxGSLfmXL/xc+OO+BZp41Za/NRDBet/BNbkRJrYzCUu0u4GU0i/ml2dtT8b9qgkw9z6Q==} engines: {node: '>= 4.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v7.0.2+ of superagent. We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing. See the releases tab for more information at . superagent@4.1.0: resolution: {integrity: sha512-FT3QLMasz0YyCd4uIi5HNe+3t/onxMyEho7C3PSqmti3Twgy2rXT4fmkTz6wRL6bTF4uzPcfkUCa8u4JWHw8Ag==} engines: {node: '>= 6.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v7.0.2+ of superagent. We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing. See the releases tab for more information at . superagent@5.3.1: resolution: {integrity: sha512-wjJ/MoTid2/RuGCOFtlacyGNxN9QLMgcpYLDQlWFIhhdJ93kNscFonGvrpAHSCVjRVj++DGCglocF7Aej1KHvQ==} engines: {node: '>= 7.0.0'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please upgrade to v7.0.2+ of superagent. We have fixed numerous issues with streams, form-data, attach(), filesystem errors not bubbling up (ENOENT on attach()), and all tests are now passing. See the releases tab for more information at . superagent@7.1.6: resolution: {integrity: sha512-gZkVCQR1gy/oUXr+kxJMLDjla434KmSOKbx5iGD30Ql+AkJQ/YlPKECJy2nhqOsHLjGHzoDTXNSjhnvWhzKk7g==} engines: {node: '>=6.4.0 <13 || >=14'} - deprecated: Please upgrade to v9.0.0+ as we have fixed a public vulnerability with formidable dependency. Note that v9.0.0+ requires Node.js v14.18.0+. See https://github.com/ladjs/superagent/pull/1800 for insight. This project is supported and maintained by the team at Forward Email @ https://forwardemail.net + deprecated: Please downgrade to v7.1.5 if you need IE/ActiveXObject support OR upgrade to v8.0.0 as we no longer support IE and published an incorrect patch version (see https://github.com/visionmedia/superagent/issues/1731) supports-color@2.0.0: resolution: {integrity: sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==} @@ -40518,7 +40518,7 @@ snapshots: pluralize@8.0.0: {} - pnpm@9.14.3: {} + pnpm@9.14.2: {} points-on-curve@0.2.0: {} @@ -43086,7 +43086,7 @@ snapshots: ts-interface-checker@0.1.13: {} - ts-jest@29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2): + ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.6.3): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 @@ -43097,15 +43097,15 @@ snapshots: lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.6.3 - typescript: 5.7.2 + typescript: 5.6.3 yargs-parser: 21.1.1 optionalDependencies: - '@babel/core': 7.26.0 + '@babel/core': 8.0.0-alpha.13 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@7.26.0) + babel-jest: 29.7.0(@babel/core@8.0.0-alpha.13) - ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.6.3): + ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 @@ -43116,7 +43116,7 @@ snapshots: lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.6.3 - typescript: 5.6.3 + typescript: 5.7.2 yargs-parser: 21.1.1 optionalDependencies: '@babel/core': 8.0.0-alpha.13 From b6e1aaf922d08b231f8ad497fc6f8438be1a2afd Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 31 Dec 2024 13:39:36 -0500 Subject: [PATCH 03/12] pnpm-lock.yaml --- pnpm-lock.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e2541dffc9f1f..451bd41f64ffc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12432,7 +12432,7 @@ importers: version: 3.1.7 ts-jest: specifier: ^29.2.5 - version: 29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2) + version: 29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2) typescript: specifier: ^5.6 version: 5.7.2 @@ -43086,7 +43086,7 @@ snapshots: ts-interface-checker@0.1.13: {} - ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.6.3): + ts-jest@29.2.5(@babel/core@7.26.0)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.26.0))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 @@ -43097,15 +43097,15 @@ snapshots: lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.6.3 - typescript: 5.6.3 + typescript: 5.7.2 yargs-parser: 21.1.1 optionalDependencies: - '@babel/core': 8.0.0-alpha.13 + '@babel/core': 7.26.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@8.0.0-alpha.13) + babel-jest: 29.7.0(@babel/core@7.26.0) - ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.7.2): + ts-jest@29.2.5(@babel/core@8.0.0-alpha.13)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@8.0.0-alpha.13))(jest@29.7.0(@types/node@20.17.6)(babel-plugin-macros@3.1.0))(typescript@5.6.3): dependencies: bs-logger: 0.2.6 ejs: 3.1.10 @@ -43116,7 +43116,7 @@ snapshots: lodash.memoize: 4.1.2 make-error: 1.3.6 semver: 7.6.3 - typescript: 5.7.2 + typescript: 5.6.3 yargs-parser: 21.1.1 optionalDependencies: '@babel/core': 8.0.0-alpha.13 From 9113d56cd6f61610b0d8f8e300e82ec26ec6e812 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 31 Dec 2024 13:57:54 -0500 Subject: [PATCH 04/12] update --- .../sources/new-collection-events/new-collection-events.mjs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/opensea/sources/new-collection-events/new-collection-events.mjs b/components/opensea/sources/new-collection-events/new-collection-events.mjs index 7abfc26c8a1e8..7d6917bd9a7fe 100644 --- a/components/opensea/sources/new-collection-events/new-collection-events.mjs +++ b/components/opensea/sources/new-collection-events/new-collection-events.mjs @@ -56,6 +56,10 @@ export default { } } + if (items?.length) { + return; + } + if (max) { items = items.slice(-1 * max); } From a46d6f3489e0e88222ebb73a59f952daf6a6cd4f Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 31 Dec 2024 14:03:20 -0500 Subject: [PATCH 05/12] fix --- .../sources/new-collection-events/new-collection-events.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/opensea/sources/new-collection-events/new-collection-events.mjs b/components/opensea/sources/new-collection-events/new-collection-events.mjs index 7d6917bd9a7fe..98926f3af0715 100644 --- a/components/opensea/sources/new-collection-events/new-collection-events.mjs +++ b/components/opensea/sources/new-collection-events/new-collection-events.mjs @@ -56,7 +56,7 @@ export default { } } - if (items?.length) { + if (!items?.length) { return; } From dd0a4698585417f7a9f47c155e188f4ce27cc805 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Fri, 3 Jan 2025 13:10:36 -0500 Subject: [PATCH 06/12] add bottleneck and refactor source --- components/opensea/opensea.app.mjs | 27 ++------ components/opensea/package.json | 3 +- .../new-collection-events.mjs | 66 +++++++++++-------- 3 files changed, 46 insertions(+), 50 deletions(-) diff --git a/components/opensea/opensea.app.mjs b/components/opensea/opensea.app.mjs index e2139c4faf764..6ea0a13f6d54c 100644 --- a/components/opensea/opensea.app.mjs +++ b/components/opensea/opensea.app.mjs @@ -1,4 +1,10 @@ import { axios } from "@pipedream/platform"; +import Bottleneck from "bottleneck"; +const limiter = new Bottleneck({ + minTime: 200, // 5 requests per second + maxConcurrent: 1, +}); +const axiosRateLimiter = limiter.wrap(axios); export default { type: "app", @@ -13,7 +19,7 @@ export default { path, ...otherOpts }) { - return axios($, { + return axiosRateLimiter($, { url: `${this._baseUrl()}${path}`, headers: { "X-API-KEY": this.$auth.api_key, @@ -29,24 +35,5 @@ export default { ...opts, }); }, - async *paginate({ - fn, - args = {}, - resourceKey, - }) { - let total = 0; - do { - const response = await fn(args); - const items = response[resourceKey]; - for (const item of items) { - yield item; - } - total = items?.length; - args.params = { - ...args?.params, - next: response?.next, - }; - } while (total && args.params?.next); - }, }, }; diff --git a/components/opensea/package.json b/components/opensea/package.json index 005f26b5aacfc..6c447aba15c4e 100644 --- a/components/opensea/package.json +++ b/components/opensea/package.json @@ -13,6 +13,7 @@ "access": "public" }, "dependencies": { - "@pipedream/platform": "^3.0.3" + "@pipedream/platform": "^3.0.3", + "bottleneck": "^2.19.5" } } diff --git a/components/opensea/sources/new-collection-events/new-collection-events.mjs b/components/opensea/sources/new-collection-events/new-collection-events.mjs index 98926f3af0715..13a18a732220c 100644 --- a/components/opensea/sources/new-collection-events/new-collection-events.mjs +++ b/components/opensea/sources/new-collection-events/new-collection-events.mjs @@ -24,11 +24,42 @@ export default { }, }, methods: { - getLastTimestamp() { - return this.db.get("lastTimestamp") || 0; + _getNextCursor() { + return this.db.get("nextCursor"); }, - setLastTimestamp(ts) { - this.db.set("lastTimestamp", ts); + _setNextCursor(nextCursor) { + this.db.set("nextCursor", nextCursor); + }, + async getPaginatedCollectionEvents() { + const args = { + collectionSlug: this.collectionSlug, + params: { + limit: 100, + next: this._getNextCursor(), + }, + }; + + let lastNextCursor, total = 0; + const results = []; + + do { + const { + listings, next, + } = await this.opensea.retrieveEvents(args); + results.push(...listings); + total = listings?.length; + lastNextCursor = args.params.next; + args.params.next = next; + } while (total && args.params?.next); + + this._setNextCursor(lastNextCursor); + return results; + }, + emitEvents(items) { + items.forEach((item) => { + const meta = this.generateMeta(item); + this.$emit(item, meta); + }); }, generateMeta(item) { return { @@ -38,37 +69,14 @@ export default { }; }, async processEvent(max) { - const lastTimestamp = this.getLastTimestamp(); - - const results = this.opensea.paginate({ - fn: this.opensea.retrieveEvents, - args: { - collectionSlug: this.collectionSlug, - }, - resourceKey: "listings", - }); - - let items = []; - for await (const result of results) { - const ts = result.protocol_data.parameters.startTime; - if (ts >= lastTimestamp) { - items.push(result); - } - } - + let items = await this.getPaginatedCollectionEvents(); if (!items?.length) { return; } - if (max) { items = items.slice(-1 * max); } - this.setLastTimestamp(items[items.length - 1].protocol_data.parameters.startTime); - - items.forEach((item) => { - const meta = this.generateMeta(item); - this.$emit(item, meta); - }); + this.emitEvents(items); }, }, hooks: { From d4a0a4e442f0176ffd2949bd41b68860b935aa52 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Fri, 3 Jan 2025 13:12:43 -0500 Subject: [PATCH 07/12] pnpm-lock.yaml --- pnpm-lock.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 451bd41f64ffc..8cb8b63c0b9a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7198,6 +7198,9 @@ importers: '@pipedream/platform': specifier: ^3.0.3 version: 3.0.3 + bottleneck: + specifier: ^2.19.5 + version: 2.19.5 components/openweather_api: dependencies: From a3d8a5ca5a983dc3293c723bc3aed0116199382d Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Mon, 6 Jan 2025 11:59:02 -0500 Subject: [PATCH 08/12] update to use correct endpoint --- components/opensea/opensea.app.mjs | 2 +- .../new-collection-events.mjs | 94 ++++++++++--------- 2 files changed, 51 insertions(+), 45 deletions(-) diff --git a/components/opensea/opensea.app.mjs b/components/opensea/opensea.app.mjs index 6ea0a13f6d54c..2be62f0ec6313 100644 --- a/components/opensea/opensea.app.mjs +++ b/components/opensea/opensea.app.mjs @@ -31,7 +31,7 @@ export default { collectionSlug, ...opts }) { return this._makeRequest({ - path: `/listings/collection/${collectionSlug}/all`, + path: `/events/collection/${collectionSlug}`, ...opts, }); }, diff --git a/components/opensea/sources/new-collection-events/new-collection-events.mjs b/components/opensea/sources/new-collection-events/new-collection-events.mjs index 13a18a732220c..c84f8f3f91a87 100644 --- a/components/opensea/sources/new-collection-events/new-collection-events.mjs +++ b/components/opensea/sources/new-collection-events/new-collection-events.mjs @@ -4,7 +4,7 @@ import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; export default { key: "opensea-new-collection-events", name: "New Collection Events", - description: "Emit new listings for a collection. [See the documentation](https://docs.opensea.io/reference/get_all_listings_on_collection_v2)", + description: "Emit new filtered events for a collection. [See the documentation](https://docs.opensea.io/reference/list_events_by_collection)", version: "0.0.4", dedupe: "unique", type: "source", @@ -22,61 +22,67 @@ export default { label: "Collection Slug", description: "Unique string to identify a collection on OpenSea. This can be found by visiting the collection on the OpenSea website and noting the last path parameter.", }, + eventType: { + type: "string[]", + options: [ + "all", + "cancel", + "listing", + "offer", + "order", + "redemption", + "sale", + "transfer", + ], + label: "Event Type", + description: "The type of event to filter by. If not provided, only sales will be returned.", + optional: true, + }, }, methods: { - _getNextCursor() { - return this.db.get("nextCursor"); - }, - _setNextCursor(nextCursor) { - this.db.set("nextCursor", nextCursor); - }, - async getPaginatedCollectionEvents() { - const args = { - collectionSlug: this.collectionSlug, - params: { - limit: 100, - next: this._getNextCursor(), - }, - }; - - let lastNextCursor, total = 0; - const results = []; - - do { - const { - listings, next, - } = await this.opensea.retrieveEvents(args); - results.push(...listings); - total = listings?.length; - lastNextCursor = args.params.next; - args.params.next = next; - } while (total && args.params?.next); - - this._setNextCursor(lastNextCursor); - return results; + _getLastTimestamp() { + return this.db.get("lastTimestamp"); }, - emitEvents(items) { - items.forEach((item) => { - const meta = this.generateMeta(item); - this.$emit(item, meta); - }); + _setLastTimestamp(ts) { + this.db.set("lastTimestamp", ts); }, - generateMeta(item) { + generateMeta(event) { return { - id: item.order_hash, - summary: `New ${item.type} ${item.chain} Listing`, - ts: item.protocol_data.parameters.startTime, + id: event.order_hash, + summary: `${event.asset.name || event.nft.name} ${this.eventType} event`, + ts: event.event_timestamp, }; }, async processEvent(max) { - let items = await this.getPaginatedCollectionEvents(); - if (!items?.length) { + const lastTimestamp = this._getLastTimestamp(); + let next = null; + let events = []; + do { + const resp = await this.opensea.retrieveEvents({ + collectionSlug: this.collectionSlug, + params: { + event_type: this.eventType, + after: lastTimestamp, + next, + }, + }); + if (!resp?.asset_events) { + break; + } + events.push(...resp.asset_events); + next = resp.next; + } while (lastTimestamp && next); + + if (!events.length) { return; } + this._setLastTimestamp(events[0].event_timestamp); if (max) { - items = items.slice(-1 * max); + events = events.slice(0, max); } - this.emitEvents(items); + events.reverse().forEach((event) => { + this.$emit(event, this.generateMeta(event)); + }); }, }, hooks: { From 8c379f79765e8c569ed35a739decbf50ed5bb2cd Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 7 Jan 2025 12:10:21 -0500 Subject: [PATCH 09/12] update summary --- .../sources/new-collection-events/new-collection-events.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/opensea/sources/new-collection-events/new-collection-events.mjs b/components/opensea/sources/new-collection-events/new-collection-events.mjs index c84f8f3f91a87..a302ce1aa7239 100644 --- a/components/opensea/sources/new-collection-events/new-collection-events.mjs +++ b/components/opensea/sources/new-collection-events/new-collection-events.mjs @@ -49,7 +49,7 @@ export default { generateMeta(event) { return { id: event.order_hash, - summary: `${event.asset.name || event.nft.name} ${this.eventType} event`, + summary: `${event.asset.name || event.nft.name}`, ts: event.event_timestamp, }; }, From d3d0d93f1bd8db13c0f0e0e83d38eb845e7b4834 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Tue, 7 Jan 2025 12:26:18 -0500 Subject: [PATCH 10/12] update eventType prop --- .../sources/new-collection-events/new-collection-events.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/opensea/sources/new-collection-events/new-collection-events.mjs b/components/opensea/sources/new-collection-events/new-collection-events.mjs index a302ce1aa7239..98d24b0973608 100644 --- a/components/opensea/sources/new-collection-events/new-collection-events.mjs +++ b/components/opensea/sources/new-collection-events/new-collection-events.mjs @@ -23,7 +23,7 @@ export default { description: "Unique string to identify a collection on OpenSea. This can be found by visiting the collection on the OpenSea website and noting the last path parameter.", }, eventType: { - type: "string[]", + type: "string", options: [ "all", "cancel", @@ -49,7 +49,7 @@ export default { generateMeta(event) { return { id: event.order_hash, - summary: `${event.asset.name || event.nft.name}`, + summary: `${event.asset?.name || event.nft?.name}`, ts: event.event_timestamp, }; }, From 260eb62be0ce13cbcaee1eabe84746180a1573a7 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 8 Jan 2025 11:42:57 -0500 Subject: [PATCH 11/12] updates --- components/opensea/package.json | 3 ++- .../new-collection-events/new-collection-events.mjs | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/components/opensea/package.json b/components/opensea/package.json index 6c447aba15c4e..f289b6d70aea7 100644 --- a/components/opensea/package.json +++ b/components/opensea/package.json @@ -14,6 +14,7 @@ }, "dependencies": { "@pipedream/platform": "^3.0.3", - "bottleneck": "^2.19.5" + "bottleneck": "^2.19.5", + "md5": "^2.3.0" } } diff --git a/components/opensea/sources/new-collection-events/new-collection-events.mjs b/components/opensea/sources/new-collection-events/new-collection-events.mjs index 98d24b0973608..519a22619bde1 100644 --- a/components/opensea/sources/new-collection-events/new-collection-events.mjs +++ b/components/opensea/sources/new-collection-events/new-collection-events.mjs @@ -1,5 +1,6 @@ import opensea from "../../opensea.app.mjs"; import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform"; +import md5 from "md5"; export default { key: "opensea-new-collection-events", @@ -35,7 +36,8 @@ export default { "transfer", ], label: "Event Type", - description: "The type of event to filter by. If not provided, only sales will be returned.", + description: "The type of event to filter by", + default: "all", optional: true, }, }, @@ -48,8 +50,8 @@ export default { }, generateMeta(event) { return { - id: event.order_hash, - summary: `${event.asset?.name || event.nft?.name}`, + id: md5(JSON.stringify(event)), + summary: `New ${event.event_type} event`, ts: event.event_timestamp, }; }, From 03ecacc7a60853cb538199bea5be4780f06957a7 Mon Sep 17 00:00:00 2001 From: michelle0927 Date: Wed, 8 Jan 2025 11:45:22 -0500 Subject: [PATCH 12/12] pnpm-lock.yaml --- pnpm-lock.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8cb8b63c0b9a2..4f0b997dc7a13 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7201,6 +7201,9 @@ importers: bottleneck: specifier: ^2.19.5 version: 2.19.5 + md5: + specifier: ^2.3.0 + version: 2.3.0 components/openweather_api: dependencies: