From d20938bdd8660477b31265ff8a899b86ed572d4c Mon Sep 17 00:00:00 2001 From: GALLLASMILAN Date: Fri, 22 Nov 2024 09:14:18 +0100 Subject: [PATCH] feat: pr review Signed-off-by: GALLLASMILAN --- .env.test | 1 + README.md | 2 +- docs/README.md | 4 ++ docs/instrumentation.md | 4 +- docs/native-telemetry.md | 2 +- examples/helpers/telemetry.js | 5 +- examples/llms/instrumentation.ts | 5 -- examples/tools/instrumentation.ts | 5 -- package.json | 17 +++--- src/instrumentation/config.ts | 5 +- .../create-telemetry-metrics-middleware.ts | 5 +- src/instrumentation/helpers/create-span.ts | 6 +- src/instrumentation/opentelemetry.ts | 12 ++-- src/instrumentation/sdk.ts | 8 +-- src/instrumentation/types.ts | 12 ++-- yarn.lock | 55 ++++++++++++++----- 16 files changed, 84 insertions(+), 64 deletions(-) create mode 100644 .env.test diff --git a/.env.test b/.env.test new file mode 100644 index 00000000..bbd96c30 --- /dev/null +++ b/.env.test @@ -0,0 +1 @@ +BEE_FRAMEWORK_INSTRUMENTATION_METRICS_ENABLED=false diff --git a/README.md b/README.md index 1d5063e4..498a632d 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,7 @@ All content in these repositories including code has been provided by IBM under ## Telemetry -Some metrics are collected by default. See the [Native Telemetry](./docs/native-telemetry.md) for more detail. +Some metrics are collected by default. See the [Native Telemetry](/docs/native-telemetry.md) for more detail. ## Contributors diff --git a/docs/README.md b/docs/README.md index 93d67885..498a632d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -211,6 +211,10 @@ This project and everyone participating in it are governed by the [Code of Condu All content in these repositories including code has been provided by IBM under the associated open source software license and IBM is under no obligation to provide enhancements, updates, or support. IBM developers produced this code as an open source project (not as an IBM product), and IBM makes no assertions as to the level of quality nor security, and will not be maintaining this code going forward. +## Telemetry + +Some metrics are collected by default. See the [Native Telemetry](/docs/native-telemetry.md) for more detail. + ## Contributors Special thanks to our contributors for helping us improve Bee Agent Framework. diff --git a/docs/instrumentation.md b/docs/instrumentation.md index f415bc31..64006a62 100644 --- a/docs/instrumentation.md +++ b/docs/instrumentation.md @@ -34,9 +34,9 @@ You can manually create spans during the `run` process to track specific parts o Example of creating a span: ```ts -import { trace } from "@opentelemetry/api"; +import { api } from "@opentelemetry/sdk-node"; -const tracer = trace.getTracer("bee-agent-framework"); +const tracer = api.trace.getTracer("bee-agent-framework"); function exampleFunction() { const span = tracer.startSpan("example-function-span"); diff --git a/docs/native-telemetry.md b/docs/native-telemetry.md index a7e5dbaf..c4584ecd 100644 --- a/docs/native-telemetry.md +++ b/docs/native-telemetry.md @@ -22,5 +22,5 @@ We value your privacy and ensure that **no sensitive data** is collected through We understand that not all users want to send telemetry data. You can easily disable this feature by setting an environment variable: ```bash -BEE_FRAMEWORK_INSTRUMENTATION_METRICS_DISABLED=true +BEE_FRAMEWORK_INSTRUMENTATION_METRICS_ENABLED=false ``` diff --git a/examples/helpers/telemetry.js b/examples/helpers/telemetry.js index 59d7120e..ecfa9320 100644 --- a/examples/helpers/telemetry.js +++ b/examples/helpers/telemetry.js @@ -1,12 +1,11 @@ import "@opentelemetry/instrumentation/hook.mjs"; -import { NodeSDK } from "@opentelemetry/sdk-node"; +import { NodeSDK, resources } from "@opentelemetry/sdk-node"; import { ConsoleSpanExporter } from "@opentelemetry/sdk-trace-node"; -import { Resource } from "@opentelemetry/resources"; import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions"; import { Version } from "bee-agent-framework/version"; const sdk = new NodeSDK({ - resource: new Resource({ + resource: new resources.Resource({ [ATTR_SERVICE_NAME]: "bee-agent-framework", [ATTR_SERVICE_VERSION]: Version, }), diff --git a/examples/llms/instrumentation.ts b/examples/llms/instrumentation.ts index 8c971e4b..22d4562e 100644 --- a/examples/llms/instrumentation.ts +++ b/examples/llms/instrumentation.ts @@ -20,8 +20,3 @@ const response = await llm.generate([ ]); logger.info(`LLM 🤖 (txt) : ${response.getTextContent()}`); - -// Wait briefly to ensure all telemetry data has been processed -setTimeout(() => { - logger.info("Process exiting after OpenTelemetry flush."); -}, 5_000); // Adjust the delay as needed diff --git a/examples/tools/instrumentation.ts b/examples/tools/instrumentation.ts index 20320818..cd8c189b 100644 --- a/examples/tools/instrumentation.ts +++ b/examples/tools/instrumentation.ts @@ -14,8 +14,3 @@ const result = await tool.run({ end_date: "2024-10-10", }); logger.info(`OpenMeteoTool 🤖 (txt) : ${result.getTextContent()}`); - -// Wait briefly to ensure all telemetry data has been processed -setTimeout(() => { - logger.info("Process exiting after OpenTelemetry flush."); -}, 5_000); // Adjust the delay as needed diff --git a/package.json b/package.json index c1c8403d..2f50ef3f 100644 --- a/package.json +++ b/package.json @@ -151,13 +151,13 @@ "lint:fix": "yarn eslint --fix", "format": "yarn prettier --check .", "format:fix": "yarn prettier --write .", - "test:unit": "BEE_FRAMEWORK_INSTRUMENTATION_METRICS_DISABLED=true vitest run src", - "test:unit:watch": "BEE_FRAMEWORK_INSTRUMENTATION_METRICS_DISABLED=true vitest run src", - "test:e2e": "BEE_FRAMEWORK_INSTRUMENTATION_METRICS_DISABLED=true vitest run tests/e2e", - "test:e2e:watch": "BEE_FRAMEWORK_INSTRUMENTATION_METRICS_DISABLED=true vitest watch tests/e2e", + "test:unit": "vitest run src", + "test:unit:watch": "vitest run src", + "test:e2e": "vitest run tests/e2e", + "test:e2e:watch": "vitest watch tests/e2e", "test:all": "yarn test:unit && yarn test:e2e && yarn test:examples", - "test:examples": "BEE_FRAMEWORK_INSTRUMENTATION_METRICS_DISABLED=true vitest --config ./examples/vitest.examples.config.ts run tests/examples", - "test:examples:watch": "BEE_FRAMEWORK_INSTRUMENTATION_METRICS_DISABLED=true vitest --config ./examples/vitest.examples.config.ts watch tests/examples", + "test:examples": "vitest --config ./examples/vitest.examples.config.ts run tests/examples", + "test:examples:watch": "vitest --config ./examples/vitest.examples.config.ts watch tests/examples", "prepare": "husky", "copyright": "./scripts/copyright.sh", "copyright:check": "TYPE=check ./scripts/copyright.sh", @@ -167,10 +167,9 @@ }, "dependencies": { "@ai-zen/node-fetch-event-source": "^2.1.4", - "@opentelemetry/api": "^1.9.0", + "@connectrpc/connect": "^1.6.1", + "@connectrpc/connect-node": "^1.6.1", "@opentelemetry/exporter-metrics-otlp-http": "^0.54.2", - "@opentelemetry/resources": "^1.27.0", - "@opentelemetry/sdk-metrics": "^1.27.0", "@opentelemetry/sdk-node": "^0.54.2", "@opentelemetry/semantic-conventions": "^1.27.0", "@streamparser/json": "^0.0.21", diff --git a/src/instrumentation/config.ts b/src/instrumentation/config.ts index 20582687..5fa958ec 100644 --- a/src/instrumentation/config.ts +++ b/src/instrumentation/config.ts @@ -18,8 +18,9 @@ import { parseEnv } from "@/internals/env.js"; import { z } from "zod"; export const INSTRUMENTATION_ENABLED = parseEnv.asBoolean("BEE_FRAMEWORK_INSTRUMENTATION_ENABLED"); -export const INSTRUMENTATION_METRICS_ENABLED = !parseEnv.asBoolean( - "BEE_FRAMEWORK_INSTRUMENTATION_METRICS_DISABLED", +export const INSTRUMENTATION_METRICS_ENABLED = parseEnv.asBoolean( + "BEE_FRAMEWORK_INSTRUMENTATION_METRICS_ENABLED", + true, ); export const INSTRUMENTATION_IGNORED_KEYS = parseEnv( diff --git a/src/instrumentation/create-telemetry-metrics-middleware.ts b/src/instrumentation/create-telemetry-metrics-middleware.ts index 7a532bdf..1a56c2f3 100644 --- a/src/instrumentation/create-telemetry-metrics-middleware.ts +++ b/src/instrumentation/create-telemetry-metrics-middleware.ts @@ -53,7 +53,10 @@ export function createTelemetryMetricsMiddleware() { // send metrics to the public collector emitter.match( (event) => event.path === `${basePath}.run.${finishEventName}`, - async () => await metricReader.forceFlush(), + async () => { + activeTracesMap.delete(traceId); + await metricReader.forceFlush(); + }, ); }; } diff --git a/src/instrumentation/helpers/create-span.ts b/src/instrumentation/helpers/create-span.ts index a7560ba5..843c5c8c 100644 --- a/src/instrumentation/helpers/create-span.ts +++ b/src/instrumentation/helpers/create-span.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { SpanStatusCode, TimeInput } from "@opentelemetry/api"; +import { api } from "@opentelemetry/sdk-node"; import { FrameworkSpan } from "@/instrumentation/types.js"; import { isEmpty } from "remeda"; @@ -22,7 +22,7 @@ interface CreateSpanProps { id: string; name: string; target: string; - startedAt: TimeInput; + startedAt: api.TimeInput; ctx?: any; data?: any; error?: string; @@ -51,7 +51,7 @@ export function createSpan({ }, parent_id: parent?.id, status: { - code: error ? SpanStatusCode.ERROR : SpanStatusCode.OK, + code: error ? api.SpanStatusCode.ERROR : api.SpanStatusCode.OK, message: error ? error : "", }, start_time: startedAt, diff --git a/src/instrumentation/opentelemetry.ts b/src/instrumentation/opentelemetry.ts index 22c61153..fb7ad930 100644 --- a/src/instrumentation/opentelemetry.ts +++ b/src/instrumentation/opentelemetry.ts @@ -15,7 +15,7 @@ */ import { Version } from "@/version.js"; -import opentelemetry, { SpanStatusCode, TimeInput } from "@opentelemetry/api"; +import { api } from "@opentelemetry/sdk-node"; import { FrameworkSpan, GeneratedResponse } from "./types.js"; import { BaseAgent } from "@/agents/base.js"; import { Tool } from "@/tools/base.js"; @@ -24,8 +24,8 @@ import os from "os"; const name = "bee-agent-framework"; -export const meter = opentelemetry.metrics.getMeter(name, Version); -export const tracer = opentelemetry.trace.getTracer(name, Version); +export const meter = api.metrics.getMeter(name, Version); +export const tracer = api.trace.getTracer(name, Version); const moduleUsageGauge = meter.createGauge("module_usage"); interface ComputeTreeProps { @@ -36,8 +36,8 @@ interface ComputeTreeProps { traceId: string; version: string; runErrorSpanKey: string; - startTime: TimeInput; - endTime: TimeInput; + startTime: api.TimeInput; + endTime: api.TimeInput; source: string; } @@ -116,7 +116,7 @@ export function buildTraceTree({ if (runErrorSpan) { activeSpan.setStatus(runErrorSpan.status); } else { - activeSpan.setStatus({ code: SpanStatusCode.OK }); + activeSpan.setStatus({ code: api.SpanStatusCode.OK }); } // set nested spans diff --git a/src/instrumentation/sdk.ts b/src/instrumentation/sdk.ts index 69e43ec7..07f47098 100644 --- a/src/instrumentation/sdk.ts +++ b/src/instrumentation/sdk.ts @@ -16,9 +16,7 @@ import { Version } from "@/version.js"; import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http"; -import { Resource } from "@opentelemetry/resources"; -import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; -import { NodeSDK } from "@opentelemetry/sdk-node"; +import { NodeSDK, metrics, resources } from "@opentelemetry/sdk-node"; import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions"; import { INSTRUMENTATION_METRICS_ENABLED } from "./config.js"; @@ -28,12 +26,12 @@ const metricExporter = new OTLPMetricExporter({ url: "https://bee-collector.apps.fmaas-backend.fmaas.res.ibm.com/v1/metrics", }); -export const metricReader = new PeriodicExportingMetricReader({ +export const metricReader = new metrics.PeriodicExportingMetricReader({ exporter: metricExporter, }); export const sdk = new NodeSDK({ - resource: new Resource({ + resource: new resources.Resource({ [ATTR_SERVICE_NAME]: "bee-agent-framework", [ATTR_SERVICE_VERSION]: Version, }), diff --git a/src/instrumentation/types.ts b/src/instrumentation/types.ts index b45b5d98..dde29d21 100644 --- a/src/instrumentation/types.ts +++ b/src/instrumentation/types.ts @@ -14,22 +14,22 @@ * limitations under the License. */ -import { Attributes, SpanStatus, TimeInput } from "@opentelemetry/api"; +import { api } from "@opentelemetry/sdk-node"; export interface FrameworkSpan { attributes: { - ctx?: Attributes | null; - data?: Attributes | null; + ctx?: api.Attributes | null; + data?: api.Attributes | null; target: string; }; context: { span_id: string; }; - end_time: TimeInput; + end_time: api.TimeInput; name: string; parent_id?: string; - start_time: TimeInput; - status: SpanStatus; + start_time: api.TimeInput; + status: api.SpanStatus; } export interface GeneratedResponse { diff --git a/yarn.lock b/yarn.lock index 776ee980..537e80da 100644 --- a/yarn.lock +++ b/yarn.lock @@ -733,6 +733,27 @@ __metadata: languageName: node linkType: hard +"@connectrpc/connect-node@npm:^1.6.1": + version: 1.6.1 + resolution: "@connectrpc/connect-node@npm:1.6.1" + dependencies: + undici: "npm:^5.28.4" + peerDependencies: + "@bufbuild/protobuf": ^1.10.0 + "@connectrpc/connect": 1.6.1 + checksum: 10c0/9891bbbe5ec155d16141e378c120dd6d4c47e1517656d4676aca762d70426a9eb3d9ec92595a7cfc4f5cbe40ff5be572d0c3d9010058107854e7f62ee05fb46e + languageName: node + linkType: hard + +"@connectrpc/connect@npm:^1.6.1": + version: 1.6.1 + resolution: "@connectrpc/connect@npm:1.6.1" + peerDependencies: + "@bufbuild/protobuf": ^1.10.0 + checksum: 10c0/35c6fd3e33c3a1ff9dce230b059ecd7991ef0dc60c16fb898e5c46b930a01077ac0b34d53d6742cc8ed079f20f8eacc7c77a8620aeec9efaf68950494f387011 + languageName: node + linkType: hard + "@conventional-changelog/git-client@npm:^1.0.0": version: 1.0.1 resolution: "@conventional-changelog/git-client@npm:1.0.1" @@ -1383,6 +1404,13 @@ __metadata: languageName: node linkType: hard +"@fastify/busboy@npm:^2.0.0": + version: 2.1.1 + resolution: "@fastify/busboy@npm:2.1.1" + checksum: 10c0/6f8027a8cba7f8f7b736718b013f5a38c0476eea67034c94a0d3c375e2b114366ad4419e6a6fa7ffc2ef9c6d3e0435d76dd584a7a1cbac23962fda7650b579e3 + languageName: node + linkType: hard + "@gar/promisify@npm:^1.0.1": version: 1.1.3 resolution: "@gar/promisify@npm:1.1.3" @@ -2302,7 +2330,7 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/api@npm:1.x, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.9.0": +"@opentelemetry/api@npm:1.x, @opentelemetry/api@npm:^1.3.0": version: 1.9.0 resolution: "@opentelemetry/api@npm:1.9.0" checksum: 10c0/9aae2fe6e8a3a3eeb6c1fdef78e1939cf05a0f37f8a4fae4d6bf2e09eb1e06f966ece85805626e01ba5fab48072b94f19b835449e58b6d26720ee19a58298add @@ -2700,18 +2728,6 @@ __metadata: languageName: node linkType: hard -"@opentelemetry/sdk-metrics@npm:^1.27.0": - version: 1.30.1 - resolution: "@opentelemetry/sdk-metrics@npm:1.30.1" - dependencies: - "@opentelemetry/core": "npm:1.30.1" - "@opentelemetry/resources": "npm:1.30.1" - peerDependencies: - "@opentelemetry/api": ">=1.3.0 <1.10.0" - checksum: 10c0/7e60178e61eaf49db5d74f6c3701706762d71d370044253c72bb5668dba3a435030ed6847605ee55d0e1b8908ad123a2517b5f00415a2fb3d98468a0a318e3c0 - languageName: node - linkType: hard - "@opentelemetry/sdk-node@npm:^0.56.0": version: 0.56.0 resolution: "@opentelemetry/sdk-node@npm:0.56.0" @@ -4799,6 +4815,8 @@ __metadata: "@aws-sdk/client-bedrock-runtime": "npm:^3.706.0" "@commitlint/cli": "npm:^19.6.0" "@commitlint/config-conventional": "npm:^19.6.0" + "@connectrpc/connect": "npm:^1.6.1" + "@connectrpc/connect-node": "npm:^1.6.1" "@elastic/elasticsearch": "npm:^8.16.2" "@eslint/js": "npm:^9.16.0" "@eslint/markdown": "npm:^6.2.1" @@ -4812,11 +4830,9 @@ __metadata: "@langchain/langgraph": "npm:^0.2.39" "@langchain/ollama": "npm:^0.1.4" "@modelcontextprotocol/sdk": "npm:^1.0.4" - "@opentelemetry/api": "npm:^1.9.0" "@opentelemetry/exporter-metrics-otlp-http": "npm:^0.54.2" "@opentelemetry/instrumentation": "npm:^0.56.0" "@opentelemetry/resources": "npm:^1.29.0" - "@opentelemetry/sdk-metrics": "npm:^1.27.0" "@opentelemetry/sdk-node": "npm:^0.56.0" "@opentelemetry/sdk-trace-node": "npm:^1.29.0" "@opentelemetry/semantic-conventions": "npm:^1.28.0" @@ -13649,6 +13665,15 @@ __metadata: languageName: node linkType: hard +"undici@npm:^5.28.4": + version: 5.28.5 + resolution: "undici@npm:5.28.5" + dependencies: + "@fastify/busboy": "npm:^2.0.0" + checksum: 10c0/4dfaa13089fe4c0758f84ec0d34b257e58608e6be3aa540f493b9864b39e3fdcd0a1ace38e434fe79db55f833aa30bcfddd8d6cbe3e0982b0dcae8ec17b65e08 + languageName: node + linkType: hard + "undici@npm:^6.21.1": version: 6.21.1 resolution: "undici@npm:6.21.1"