From 3ab7037717b21877088c2b17291d2c8f3264350b Mon Sep 17 00:00:00 2001 From: xzchaoo Date: Wed, 25 Oct 2023 14:24:15 +0800 Subject: [PATCH] feat: agg metric name format (#712) --- .../executor/MetricStorageOutput.java | 19 ++++---- server/agg/agg-executor-api/pom.xml | 4 ++ .../v1/executor/output/AggStringLookup.java | 48 +++++++++++++++++++ .../v1/executor/output/XConsoleOutput.java | 42 +++++++++++++--- server/holoinsight-dependencies/pom.xml | 6 +++ 5 files changed, 105 insertions(+), 14 deletions(-) create mode 100644 server/agg/agg-executor-api/src/main/java/io/holoinsight/server/agg/v1/executor/output/AggStringLookup.java diff --git a/server/agg/agg-boot/src/main/java/io/holoinsight/server/agg/v1/executor/executor/MetricStorageOutput.java b/server/agg/agg-boot/src/main/java/io/holoinsight/server/agg/v1/executor/executor/MetricStorageOutput.java index fb6769234..5ea658500 100644 --- a/server/agg/agg-boot/src/main/java/io/holoinsight/server/agg/v1/executor/executor/MetricStorageOutput.java +++ b/server/agg/agg-boot/src/main/java/io/holoinsight/server/agg/v1/executor/executor/MetricStorageOutput.java @@ -9,12 +9,14 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.text.StringSubstitutor; import org.springframework.beans.factory.annotation.Autowired; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.serializer.SerializerFeature; import io.holoinsight.server.agg.v1.core.Utils; +import io.holoinsight.server.agg.v1.executor.output.AggStringLookup; import io.holoinsight.server.agg.v1.executor.output.MergedCompleteness; import io.holoinsight.server.agg.v1.executor.output.XOutput; import io.holoinsight.server.extension.MetricStorage; @@ -47,7 +49,10 @@ public void write(Batch batch) { long time0 = System.currentTimeMillis(); String baseName = batch.oi.getName(); - boolean useFormatName = baseName.contains("%s"); + AggStringLookup aggStringLookup = new AggStringLookup(); + boolean useFormatName = baseName.contains("$"); + StringSubstitutor stringSubstitutor = new StringSubstitutor(aggStringLookup); + String ts = Utils.formatTimeShort(batch.window.getTimestamp()); boolean debug = false; @@ -63,15 +68,13 @@ public void write(Batch batch) { } String metricName; - if ("value".equals(e.getKey())) { - metricName = baseName; + if (useFormatName) { + aggStringLookup.bind(e.getKey(), g.tags, batch.key.getPartitionInfo()); + metricName = stringSubstitutor.replace(baseName); } else { - if (useFormatName) { - metricName = String.format(baseName, e.getKey()); - } else { - metricName = baseName + "_" + e.getKey(); - } + metricName = baseName; } + usedMetricNames.add(metricName); try { diff --git a/server/agg/agg-executor-api/pom.xml b/server/agg/agg-executor-api/pom.xml index 70c26dba8..b53e10644 100644 --- a/server/agg/agg-executor-api/pom.xml +++ b/server/agg/agg-executor-api/pom.xml @@ -22,6 +22,10 @@ com.alibaba fastjson + + org.apache.commons + commons-text + diff --git a/server/agg/agg-executor-api/src/main/java/io/holoinsight/server/agg/v1/executor/output/AggStringLookup.java b/server/agg/agg-executor-api/src/main/java/io/holoinsight/server/agg/v1/executor/output/AggStringLookup.java new file mode 100644 index 000000000..933cdf4ff --- /dev/null +++ b/server/agg/agg-executor-api/src/main/java/io/holoinsight/server/agg/v1/executor/output/AggStringLookup.java @@ -0,0 +1,48 @@ +/* + * Copyright 2022 Holoinsight Project Authors. Licensed under Apache-2.0. + */ +package io.holoinsight.server.agg.v1.executor.output; + +import java.util.Map; + +import org.apache.commons.text.lookup.StringLookup; + +import io.holoinsight.server.agg.v1.executor.executor.FixedSizeTags; + +/** + *

+ * created at 2023/10/25 + * + * @author xzchaoo + */ +public class AggStringLookup implements StringLookup { + + private static final String UNKNOWN = "UNKNOWN"; + + private String fieldName; + private FixedSizeTags tags; + private Map partition; + + public void bind(String fieldName, FixedSizeTags tags, Map partition) { + this.fieldName = fieldName; + this.tags = tags; + this.partition = partition; + } + + @Override + public String lookup(String key) { + if (fieldName != null && "field".equals(key)) { + return fieldName; + } + + if (tags != null && key.startsWith("tag.")) { + return tags.getTagValue(key.substring("tag.".length()), UNKNOWN); + } + + if (partition != null && key.startsWith("partition.")) { + return partition.getOrDefault(key.substring("partition.".length()), UNKNOWN); + } + + return UNKNOWN; + } +} diff --git a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/output/XConsoleOutput.java b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/output/XConsoleOutput.java index 0d573cfec..9dd01bef4 100644 --- a/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/output/XConsoleOutput.java +++ b/server/agg/agg-executor/src/main/java/io/holoinsight/server/agg/v1/executor/output/XConsoleOutput.java @@ -5,6 +5,8 @@ import java.util.Map; +import org.apache.commons.text.StringSubstitutor; + import com.alibaba.fastjson.JSON; import io.holoinsight.server.agg.v1.core.Utils; @@ -21,6 +23,7 @@ @Slf4j public class XConsoleOutput implements XOutput { public static final String TYPE = "CONSOLE"; + private boolean printMultiFields = true; @Override public String type() { @@ -32,15 +35,42 @@ public void write(Batch batch) { AggTaskKey key = batch.key; WindowInfo w = batch.window; OutputItem oi = batch.oi; + + String baseName = batch.oi.getName(); + AggStringLookup aggStringLookup = new AggStringLookup(); + boolean useFormatName = baseName.contains("$"); + StringSubstitutor stringSubstitutor = new StringSubstitutor(aggStringLookup); + for (Group g : batch.getGroups()) { Map finalFields = g.finalFields; - log.info("[agg] [{}] emit name=[{}] ts=[{}] tags=[{}] fields={}", // - key, // - oi.getName(), // - Utils.formatTimeShort(w.getTimestamp()), // - g.getTags(), // - JSON.toJSONString(finalFields)); // + + if (printMultiFields) { + log.info("[agg] [{}] emit name=[{}] ts=[{}] tags=[{}] fields={}", // + key, // + oi.getName(), // + Utils.formatTimeShort(w.getTimestamp()), // + g.getTags(), // + JSON.toJSONString(finalFields)); // + } else { + for (Map.Entry e : finalFields.entrySet()) { + String metricName; + if (useFormatName) { + aggStringLookup.bind(e.getKey(), g.tags, batch.key.getPartitionInfo()); + metricName = stringSubstitutor.replace(baseName); + } else { + metricName = baseName; + } + log.info("[agg] [{}] emit name=[{}] ts=[{}] metric=[{}] tags=[{}] value=[{}]", // + key, // + oi.getName(), // + Utils.formatTimeShort(w.getTimestamp()), // + metricName, g.getTags(), // + e.getValue()); // + } + } + } + } } diff --git a/server/holoinsight-dependencies/pom.xml b/server/holoinsight-dependencies/pom.xml index 1a599c056..a19138fb2 100644 --- a/server/holoinsight-dependencies/pom.xml +++ b/server/holoinsight-dependencies/pom.xml @@ -254,6 +254,12 @@ hll 1.6.0 + + + org.apache.commons + commons-text + 1.10.0 +