From 881ccc9383e55bd944a4f4fd317d825429cb06b3 Mon Sep 17 00:00:00 2001 From: islxyqwe Date: Thu, 19 Oct 2023 11:07:09 +0800 Subject: [PATCH] fix: computed field with filter (#192) * fix: inner field cannot move to filter & meta * fix: field stat with computed field * feat: workflow with computed field * fix: computed filter not in graph --- .../graphic-walker/src/computation/index.ts | 47 ++++++-- .../src/fields/datasetFields/utils.ts | 113 +++++++++++------- .../fields/filterField/filterEditDialog.tsx | 3 +- .../src/fields/filterField/tabs.tsx | 2 +- .../src/store/visualSpecStore.ts | 10 +- packages/graphic-walker/src/utils/workflow.ts | 81 +++++++------ 6 files changed, 163 insertions(+), 93 deletions(-) diff --git a/packages/graphic-walker/src/computation/index.ts b/packages/graphic-walker/src/computation/index.ts index 5f8d6bf8..ca6c9d0a 100644 --- a/packages/graphic-walker/src/computation/index.ts +++ b/packages/graphic-walker/src/computation/index.ts @@ -1,4 +1,13 @@ -import type { IComputationFunction, IDataQueryPayload, IDataQueryWorkflowStep, IDatasetStats, IFieldStats, IRow } from '../interfaces'; +import type { + IComputationFunction, + IDataQueryPayload, + IDataQueryWorkflowStep, + IDatasetStats, + IField, + IFieldStats, + IRow, + ITransformWorkflowStep, +} from '../interfaces'; import { getTimeFormat } from '../lib/inferMeta'; export const datasetStats = async (service: IComputationFunction): Promise => { @@ -63,19 +72,33 @@ export const dataQuery = async (service: IComputationFunction, workflow: IDataQu return res; }; -export const fieldStat = async (service: IComputationFunction, field: string, options: { values?: boolean; range?: boolean }): Promise => { +export const fieldStat = async (service: IComputationFunction, field: IField, options: { values?: boolean; range?: boolean }): Promise => { const { values = true, range = true } = options; - const COUNT_ID = `count_${field}`; - const MIN_ID = `min_${field}`; - const MAX_ID = `max_${field}`; + const COUNT_ID = `count_${field.fid}`; + const MIN_ID = `min_${field.fid}`; + const MAX_ID = `max_${field.fid}`; + const transformWork: ITransformWorkflowStep[] = field.computed + ? [ + { + type: 'transform', + transform: [ + { + expression: field.expression!, + key: field.fid, + }, + ], + }, + ] + : []; const valuesQueryPayload: IDataQueryPayload = { workflow: [ + ...transformWork, { type: 'view', query: [ { op: 'aggregate', - groupBy: [field], + groupBy: [field.fid], measures: [ { field: '*', @@ -91,6 +114,7 @@ export const fieldStat = async (service: IComputationFunction, field: string, op const valuesRes = values ? await service(valuesQueryPayload) : []; const rangeQueryPayload: IDataQueryPayload = { workflow: [ + ...transformWork, { type: 'view', query: [ @@ -99,12 +123,12 @@ export const fieldStat = async (service: IComputationFunction, field: string, op groupBy: [], measures: [ { - field, + field: field.fid, agg: 'min', asFieldKey: MIN_ID, }, { - field, + field: field.fid, agg: 'max', asFieldKey: MAX_ID, }, @@ -132,14 +156,13 @@ export const fieldStat = async (service: IComputationFunction, field: string, op values: valuesRes .sort((a, b) => b[COUNT_ID] - a[COUNT_ID]) .map((row) => ({ - value: row[field], + value: row[field.fid], count: row[COUNT_ID], })), range: [rangeRes[MIN_ID], rangeRes[MAX_ID]], }; }; - export async function getSample(service: IComputationFunction, field: string) { const res = await service({ workflow: [ @@ -183,7 +206,7 @@ export async function getTemporalRange(service: IComputationFunction, field: str field, agg: 'max', asFieldKey: MAX_ID, - format + format, }, ], }, @@ -196,6 +219,6 @@ export async function getTemporalRange(service: IComputationFunction, field: str [MIN_ID]: 0, [MAX_ID]: 0, }, - ] = await service(rangeQueryPayload); + ] = await service(rangeQueryPayload); return [new Date(rangeRes[MIN_ID]).getTime(), new Date(rangeRes[MAX_ID]).getTime()] as [number, number]; } diff --git a/packages/graphic-walker/src/fields/datasetFields/utils.ts b/packages/graphic-walker/src/fields/datasetFields/utils.ts index 75f2593d..aa3e7a4f 100644 --- a/packages/graphic-walker/src/fields/datasetFields/utils.ts +++ b/packages/graphic-walker/src/fields/datasetFields/utils.ts @@ -1,128 +1,155 @@ -import { useMemo } from "react"; -import { useTranslation } from "react-i18next"; -import { useCompututaion, useVizStore } from "../../store"; -import type { IActionMenuItem } from "../../components/actionMenu/list"; -import { COUNT_FIELD_ID, DATE_TIME_DRILL_LEVELS, DATE_TIME_FEATURE_LEVELS } from "../../constants"; -import { getSample } from "../../computation"; -import { getTimeFormat } from "../../lib/inferMeta"; - +import { useMemo } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useCompututaion, useVizStore } from '../../store'; +import type { IActionMenuItem } from '../../components/actionMenu/list'; +import { COUNT_FIELD_ID, DATE_TIME_DRILL_LEVELS, DATE_TIME_FEATURE_LEVELS, MEA_KEY_ID, MEA_VAL_ID } from '../../constants'; +import { getSample } from '../../computation'; +import { getTimeFormat } from '../../lib/inferMeta'; const keepTrue = (array: (T | 0 | null | false | undefined | void)[]): T[] => { return array.filter(Boolean) as T[]; }; -export const useMenuActions = (channel: "dimensions" | "measures"): IActionMenuItem[][] => { +export const useMenuActions = (channel: 'dimensions' | 'measures'): IActionMenuItem[][] => { const vizStore = useVizStore(); const fields = vizStore.currentVis.encodings[channel]; - const { t } = useTranslation('translation', { keyPrefix: "field_menu" }); + const { t } = useTranslation('translation', { keyPrefix: 'field_menu' }); const computation = useCompututaion(); return useMemo(() => { return fields.map((f, index) => { const isDateTimeDrilled = f.expression?.op === 'dateTimeDrill'; - + const isInnerField = [COUNT_FIELD_ID, MEA_KEY_ID, MEA_VAL_ID].includes(f.fid); return keepTrue([ channel === 'dimensions' && { label: t('to_mea'), + disabled: isInnerField, onPress() { - vizStore.moveField("dimensions", index, "measures", vizStore.viewMeasures.length); + vizStore.moveField('dimensions', index, 'measures', vizStore.viewMeasures.length); }, }, channel === 'measures' && { label: t('to_dim'), - disabled: f.fid === COUNT_FIELD_ID, + disabled: isInnerField, onPress() { - vizStore.moveField("measures", index, "dimensions", vizStore.viewDimensions.length); + vizStore.moveField('measures', index, 'dimensions', vizStore.viewDimensions.length); }, }, { label: t('new_calc'), - disabled: f.semanticType === 'nominal' || f.semanticType === 'ordinal', + disabled: f.semanticType === 'nominal' || f.semanticType === 'ordinal' || isInnerField, children: [ { label: t('bin'), onPress() { - vizStore.createBinField(channel, index, "bin"); + vizStore.createBinField(channel, index, 'bin'); }, }, { label: t('binCount'), disabled: f.semanticType === 'nominal' || f.semanticType === 'ordinal', onPress() { - vizStore.createBinField(channel, index, "binCount"); + vizStore.createBinField(channel, index, 'binCount'); }, }, { - label: t('log', {base: 10}), + label: t('log', { base: 10 }), disabled: f.semanticType === 'nominal' || f.semanticType === 'ordinal', onPress() { - vizStore.createLogField(channel, index, "log", 10); + vizStore.createLogField(channel, index, 'log', 10); }, }, { - label: t('log', {base: 2}), + label: t('log', { base: 2 }), disabled: f.semanticType === 'nominal' || f.semanticType === 'ordinal', onPress() { - vizStore.createLogField(channel, index, "log", 2); + vizStore.createLogField(channel, index, 'log', 2); }, }, { - label:t('binCustom'), + label: t('binCustom'), disabled: f.semanticType === 'nominal' || f.semanticType === 'ordinal', - onPress(){ + onPress() { vizStore.setShowBinSettingPanel(true); - vizStore.setCreateField({channel:channel,index:index}); - } + vizStore.setCreateField({ channel: channel, index: index }); + }, }, { - label:t('logCustom'), + label: t('logCustom'), disabled: f.semanticType === 'nominal' || f.semanticType === 'ordinal', - onPress(){ + onPress() { vizStore.setShowLogSettingPanel(true); - vizStore.setCreateField({channel:channel,index:index}) - } + vizStore.setCreateField({ channel: channel, index: index }); + }, }, - ], }, { label: t('semantic_type.name'), - children: (["nominal", "ordinal", "quantitative", "temporal"] as const).map(x => ({ + disabled: isInnerField, + children: (['nominal', 'ordinal', 'quantitative', 'temporal'] as const).map((x) => ({ label: t(`semantic_type.types.${x}`), disabled: f.semanticType === x, onPress() { const originChannel = f.analyticType === 'dimension' ? 'dimensions' : 'measures'; vizStore.changeSemanticType(originChannel, index, x); }, - })) + })), }, (f.semanticType === 'temporal' || isDateTimeDrilled) && { label: t('drill.name'), - children: DATE_TIME_DRILL_LEVELS.map(level => ({ + disabled: isInnerField, + children: DATE_TIME_DRILL_LEVELS.map((level) => ({ label: t(`drill.levels.${level}`), - disabled: isDateTimeDrilled && f.expression?.params.find(p => p.type === 'value')?.value === level, + disabled: isDateTimeDrilled && f.expression?.params.find((p) => p.type === 'value')?.value === level, onPress() { - const originField = (isDateTimeDrilled ? vizStore.allFields.find(field => field.fid === f.expression?.params.find(p => p.type === 'field')?.value) : null) ?? f; + const originField = + (isDateTimeDrilled + ? vizStore.allFields.find((field) => field.fid === f.expression?.params.find((p) => p.type === 'field')?.value) + : null) ?? f; const originChannel = originField.analyticType === 'dimension' ? 'dimensions' : 'measures'; - const originIndex = vizStore.allFields.findIndex(x => x.fid === originField.fid); - getSample(computation, originField.fid).then(getTimeFormat).then(format => vizStore.createDateTimeDrilledField(originChannel, originIndex, level, `${t(`drill.levels.${level}`)} (${originField.name || originField.fid})`, format)); + const originIndex = vizStore.allFields.findIndex((x) => x.fid === originField.fid); + getSample(computation, originField.fid) + .then(getTimeFormat) + .then((format) => + vizStore.createDateTimeDrilledField( + originChannel, + originIndex, + level, + `${t(`drill.levels.${level}`)} (${originField.name || originField.fid})`, + format + ) + ); }, })), }, (f.semanticType === 'temporal' || isDateTimeDrilled) && { label: t('drill.feature_name'), - children: DATE_TIME_FEATURE_LEVELS.map(level => ({ + disabled: isInnerField, + children: DATE_TIME_FEATURE_LEVELS.map((level) => ({ label: t(`drill.levels.${level}`), - disabled: isDateTimeDrilled && f.expression?.params.find(p => p.type === 'value')?.value === level, + disabled: isDateTimeDrilled && f.expression?.params.find((p) => p.type === 'value')?.value === level, onPress() { - const originField = (isDateTimeDrilled ? vizStore.allFields.find(field => field.fid === f.expression?.params.find(p => p.type === 'field')?.value) : null) ?? f; + const originField = + (isDateTimeDrilled + ? vizStore.allFields.find((field) => field.fid === f.expression?.params.find((p) => p.type === 'field')?.value) + : null) ?? f; const originChannel = originField.analyticType === 'dimension' ? 'dimensions' : 'measures'; - const originIndex = vizStore.allFields.findIndex(x => x.fid === originField.fid); - getSample(computation, originField.fid).then(getTimeFormat).then(format => vizStore.createDateFeatureField(originChannel, originIndex, level,`${t(`drill.levels.${level}`)} [${originField.name || originField.fid}]`, format)); + const originIndex = vizStore.allFields.findIndex((x) => x.fid === originField.fid); + getSample(computation, originField.fid) + .then(getTimeFormat) + .then((format) => + vizStore.createDateFeatureField( + originChannel, + originIndex, + level, + `${t(`drill.levels.${level}`)} [${originField.name || originField.fid}]`, + format + ) + ); }, })), }, - ]); }); }, [channel, fields, vizStore, t, computation]); diff --git a/packages/graphic-walker/src/fields/filterField/filterEditDialog.tsx b/packages/graphic-walker/src/fields/filterField/filterEditDialog.tsx index eef13ff4..569a08c5 100644 --- a/packages/graphic-walker/src/fields/filterField/filterEditDialog.tsx +++ b/packages/graphic-walker/src/fields/filterField/filterEditDialog.tsx @@ -9,6 +9,7 @@ import Tabs, { RuleFormProps } from './tabs'; import DefaultButton from '../../components/button/default'; import PrimaryButton from '../../components/button/primary'; import DropdownSelect from '../../components/dropdownSelect'; +import { COUNT_FIELD_ID, MEA_KEY_ID, MEA_VAL_ID } from '../../constants'; const QuantitativeRuleForm: React.FC = ({ rawFields, field, onChange }) => { return ; @@ -72,7 +73,7 @@ const FilterEditDialog: React.FC = observer(() => { }, [editingFilterIdx, uncontrolledField?.rule, vizStore]); const allFieldOptions = React.useMemo(() => { - return allFields.map((d) => ({ + return allFields.filter(x => ![COUNT_FIELD_ID, MEA_KEY_ID, MEA_VAL_ID].includes(x.fid)).map((d) => ({ label: d.name, value: d.fid, })); diff --git a/packages/graphic-walker/src/fields/filterField/tabs.tsx b/packages/graphic-walker/src/fields/filterField/tabs.tsx index c3f3b6e6..e70aa887 100644 --- a/packages/graphic-walker/src/fields/filterField/tabs.tsx +++ b/packages/graphic-walker/src/fields/filterField/tabs.tsx @@ -160,7 +160,7 @@ const useFieldStats = ( React.useEffect(() => { setLoading(true); let isCancelled = false; - fieldStat(computation, fid, { values, range }) + fieldStat(computation, field, { values, range }) .then((stats) => { if (isCancelled) { return; diff --git a/packages/graphic-walker/src/store/visualSpecStore.ts b/packages/graphic-walker/src/store/visualSpecStore.ts index b36ef1ed..edb85993 100644 --- a/packages/graphic-walker/src/store/visualSpecStore.ts +++ b/packages/graphic-walker/src/store/visualSpecStore.ts @@ -39,7 +39,7 @@ import { IChartForExport, } from '../interfaces'; import { GLOBAL_CONFIG } from '../config'; -import { DATE_TIME_DRILL_LEVELS, DATE_TIME_FEATURE_LEVELS } from '../constants'; +import { COUNT_FIELD_ID, DATE_TIME_DRILL_LEVELS, DATE_TIME_FEATURE_LEVELS, MEA_KEY_ID, MEA_VAL_ID } from '../constants'; import { toWorkflow } from '../utils/workflow'; import { KVTuple, uniqueId } from '../models/utils'; @@ -246,6 +246,10 @@ export class VizSpecStore { } private appendFilter(index: number, sourceKey: keyof Omit, sourceIndex: number) { + const oriF = this.currentEncodings[sourceKey][sourceIndex]; + if (oriF.fid === MEA_KEY_ID || oriF.fid === MEA_VAL_ID || oriF.fid === COUNT_FIELD_ID) { + return; + } this.visList[this.visIndex] = performers.appendFilter(this.visList[this.visIndex], index, sourceKey, sourceIndex, uniqueId()); this.editingFilterIdx = index; } @@ -338,8 +342,12 @@ export class VizSpecStore { } else if (destinationKey === 'filters') { return this.appendFilter(destinationIndex, sourceKey, sourceIndex); } + const oriF = this.currentEncodings[sourceKey][sourceIndex]; const sourceMeta = GLOBAL_CONFIG.META_FIELD_KEYS.includes(sourceKey); const destMeta = GLOBAL_CONFIG.META_FIELD_KEYS.includes(destinationKey); + if (destMeta && (oriF.fid === MEA_KEY_ID || oriF.fid === MEA_VAL_ID || oriF.fid === COUNT_FIELD_ID)) { + return; + } const limit = GLOBAL_CONFIG.CHANNEL_LIMIT[destinationKey] ?? Infinity; if (destMeta === sourceMeta) { this.visList[this.visIndex] = performers.moveField(this.visList[this.visIndex], sourceKey, sourceIndex, destinationKey, destinationIndex, limit); diff --git a/packages/graphic-walker/src/utils/workflow.ts b/packages/graphic-walker/src/utils/workflow.ts index fb3e7749..0c17e7b9 100644 --- a/packages/graphic-walker/src/utils/workflow.ts +++ b/packages/graphic-walker/src/utils/workflow.ts @@ -8,6 +8,7 @@ import type { IVisFilter, ISortWorkflowStep, IDataQueryPayload, + IFilterField, } from '../interfaces'; import type { VizSpecStore } from '../store/visualSpecStore'; import { getMeaAggKey } from '.'; @@ -65,49 +66,50 @@ export const toWorkflow = ( viewDimensions.push(...newFields.filter((x) => x?.analyticType === 'dimension')); viewMeasures.push(...newFields.filter((x) => x?.analyticType === 'measure')); } - const viewKeys = new Set([...viewDimensions, ...viewMeasures].map((f) => f.fid)); + const viewKeys = new Set([...viewDimensions, ...viewMeasures, ...viewFilters].map((f) => f.fid)); let filterWorkflow: IFilterWorkflowStep | null = null; let transformWorkflow: ITransformWorkflowStep | null = null; + let computedWorkflow: IFilterWorkflowStep | null = null; let viewQueryWorkflow: IViewWorkflowStep | null = null; let sortWorkflow: ISortWorkflowStep | null = null; // TODO: apply **fold** before filter + const createFilter = (f: IFilterField): IVisFilter => { + viewKeys.add(f.fid); + const rule = f.rule!; + if (rule.type === 'one of') { + return { + fid: f.fid, + rule: { + type: 'one of', + value: [...rule.value], + }, + }; + } else if (rule.type === 'temporal range') { + const range = [new Date(rule.value[0]).getTime(), new Date(rule.value[1]).getTime()] as const; + return { + fid: f.fid, + rule: { + type: 'temporal range', + value: range, + }, + }; + } else { + const range = [Number(rule.value[0]), Number(rule.value[1])] as const; + return { + fid: f.fid, + rule: { + type: 'range', + value: range, + }, + }; + } + }; + // First, to apply filters on the detailed data - const filters = viewFilters - .filter((f) => f.rule) - .map((f) => { - viewKeys.add(f.fid); - const rule = f.rule!; - if (rule.type === 'one of') { - return { - fid: f.fid, - rule: { - type: 'one of', - value: [...rule.value], - }, - }; - } else if (rule.type === 'temporal range') { - const range = [new Date(rule.value[0]).getTime(), new Date(rule.value[1]).getTime()] as const; - return { - fid: f.fid, - rule: { - type: 'temporal range', - value: range, - }, - }; - } else { - const range = [Number(rule.value[0]), Number(rule.value[1])] as const; - return { - fid: f.fid, - rule: { - type: 'range', - value: range, - }, - }; - } - }); + const filters = viewFilters.filter((f) => !f.computed && f.rule).map(createFilter); if (filters.length) { filterWorkflow = { type: 'filter', @@ -132,6 +134,15 @@ export const toWorkflow = ( }; } + // Third, apply filter on the transformed data + const computedFilters = viewFilters.filter((f) => f.computed && f.rule).map(createFilter); + if (computedFilters.length) { + computedWorkflow = { + type: 'filter', + filters: computedFilters, + }; + } + // Finally, to apply the aggregation // When aggregation is enabled, there're 2 cases: // 1. If any of the measures is aggregated, then we apply the aggregation @@ -174,7 +185,7 @@ export const toWorkflow = ( }; } - const steps: IDataQueryWorkflowStep[] = [filterWorkflow!, transformWorkflow!, viewQueryWorkflow!, sortWorkflow!].filter(Boolean); + const steps: IDataQueryWorkflowStep[] = [filterWorkflow!, transformWorkflow!, computedWorkflow!, viewQueryWorkflow!, sortWorkflow!].filter(Boolean); return steps; };