From fb666a4870a1745a61c26b884b943c0f8cb4a063 Mon Sep 17 00:00:00 2001 From: Howard Wu Date: Fri, 24 Feb 2023 11:37:32 +1300 Subject: [PATCH 1/2] Fix station filter logic for nodes having no children. --- cmd/fdsn-ws/fdsn_station.go | 141 ++++++++--------- cmd/fdsn-ws/fdsn_station_test.go | 253 ++++++++++++++++++++++++++++++- 2 files changed, 318 insertions(+), 76 deletions(-) diff --git a/cmd/fdsn-ws/fdsn_station.go b/cmd/fdsn-ws/fdsn_station.go index e9833208..a73527bf 100644 --- a/cmd/fdsn-ws/fdsn_station.go +++ b/cmd/fdsn-ws/fdsn_station.go @@ -575,6 +575,10 @@ func (r *FDSNStationXML) marshalText(levelVal int) *bytes.Buffer { net.StartDate.MarshalFormatText(), net.EndDate.MarshalFormatText(), net.TotalNumberStations)) } else { + if levelVal == STATION_LEVEL_STATION && len(net.Station) == 0 { + // Write Network name only + by.WriteString(fmt.Sprintf("%s|||||||\n", net.Code)) + } for s := 0; s < len(net.Station); s++ { sta := &net.Station[s] if levelVal == STATION_LEVEL_STATION { @@ -583,6 +587,10 @@ func (r *FDSNStationXML) marshalText(levelVal int) *bytes.Buffer { sta.Latitude.Value, sta.Longitude.Value, sta.Elevation.Value, sta.Site.Name, sta.StartDate.MarshalFormatText(), sta.EndDate.MarshalFormatText())) } else { + if len(sta.Channel) == 0 { + // Write Station name only + by.WriteString(fmt.Sprintf("%s|%s||||||\n", net.Code, sta.Code)) + } for c := 0; c < len(sta.Channel); c++ { cha := &sta.Channel[c] var frequency float64 @@ -610,18 +618,16 @@ func (r *FDSNStationXML) marshalText(levelVal int) *bytes.Buffer { } func (r *FDSNStationXML) doFilter(params []fdsnStationV1Search) bool { - ns := make([]NetworkType, 0) - + resultNetworks := make([]NetworkType, 0) for _, n := range r.Network { if n.doFilter(params) { - ns = append(ns, n) + resultNetworks = append(resultNetworks, n) } } - r.Network = ns + r.Network = resultNetworks - if len(ns) == 0 { - // No result ( no "Network" node ) + if len(resultNetworks) == 0 { return false } @@ -638,7 +644,7 @@ func (r *FDSNStationXML) doFilter(params []fdsnStationV1Search) bool { func (n *NetworkType) doFilter(params []fdsnStationV1Search) bool { n.TotalNumberStations = len(n.Station) matchedParams := make([]fdsnStationV1Search, 0) - ss := make([]StationType, 0) + resultStations := make([]StationType, 0) for _, p := range params { if !p.validStartEnd(time.Time(n.StartDate), time.Time(n.EndDate), STATION_LEVEL_NETWORK) { @@ -655,46 +661,34 @@ func (n *NetworkType) doFilter(params []fdsnStationV1Search) bool { return false } + if len(n.Station) == 0 { + // for network nodes without children (though unlikely to happen): + // 1. If the query parameter stops at network level, then the match is done + // 2. Otherwise, we're unable to get a match because of empty children (thus returning false) + for _, p := range matchedParams { + if p.StationReg == nil && p.ChannelReg == nil && p.LocationReg == nil { + return true + } + } + return false + } + for _, s := range n.Station { if s.doFilter(matchedParams) { - ss = append(ss, s) - } - } - - if len(ss) == 0 { - // Special case: when requested level is deeper than this level, - // but no child node from this node, then we should skip this node. - if params[0].LevelValue > STATION_LEVEL_NETWORK { - return false - } - - // NOTE: the long description under is unlikely to happen since we only got 1 network. - // However I still included the logic. - // --- - // Normally this network is included since it has met the query parameters for network. - // However, there's another case: - // e.g. "query?station=ZZZZ&level=network" - // This kind of query makes the network test bypassed due to no query parameter for network, - // but when there's no child node for this network we should skip this network. - // In short: - // when there's no child node and there's query parameter for station, channel or location, - // this network is excluded. - for _, p := range params { - if p.StationReg != nil || p.ChannelReg != nil || p.LocationReg != nil { - return false - } + resultStations = append(resultStations, s) } } - n.SelectedNumberStations = len(ss) - n.Station = ss + n.SelectedNumberStations = len(resultStations) + n.Station = resultStations - return true + // this node only valid if the children matches any query + return n.SelectedNumberStations > 0 } func (s *StationType) doFilter(params []fdsnStationV1Search) bool { s.TotalNumberChannels = len(s.Channel) - cs := make([]ChannelType, 0) + resultChannels := make([]ChannelType, 0) matchedParams := make([]fdsnStationV1Search, 0) @@ -705,10 +699,10 @@ func (s *StationType) doFilter(params []fdsnStationV1Search) bool { if p.StationReg != nil && !matchAnyRegex(s.Code, p.StationReg) { continue } - if !p.validLatLng(s.Latitude.Value, s.Longitude.Value) { + if !p.validLatLng(s.Latitude, s.Longitude) { continue } - if !p.validBounding(s.Latitude.Value, s.Longitude.Value) { + if !p.validBounding(s.Latitude, s.Longitude) { continue } matchedParams = append(matchedParams, p) @@ -719,38 +713,30 @@ func (s *StationType) doFilter(params []fdsnStationV1Search) bool { return false } - for _, c := range s.Channel { - if c.doFilter(matchedParams) { - cs = append(cs, c) + if len(s.Channel) == 0 { + // for station nodes without children: + // 1. If the query parameter stops at station level, then the match is done + // 2. Otherwise, we're unable to get a match (thus returning false) + for _, p := range matchedParams { + if p.ChannelReg == nil && p.LocationReg == nil { + return true + } } - } - if len(cs) == 0 { - // Special case: when requested level is deeper than this level, - // but no child node from this node, then we should skip this node. - if params[0].LevelValue > STATION_LEVEL_STATION { - return false - } + return false + } - // Normally this stations is included since it has met the query parameters for station. - // However, there's another case: - // e.g. "query?channel=BTT" - // This kind of query makes the station test bypassed due to no query parameter for station, - // but when there's no child node for this station we should skip this station. - // In conclusion: - // when there's no sub child and there's query parameter for channel or location, - // this station is excluded. - for _, p := range params { - if p.ChannelReg != nil || p.LocationReg != nil { - return false - } + for _, c := range s.Channel { + if c.doFilter(matchedParams) { + resultChannels = append(resultChannels, c) } } - s.SelectedNumberChannels = len(cs) - s.Channel = cs + s.SelectedNumberChannels = len(resultChannels) + s.Channel = resultChannels - return true + // this node only valid if the children matches any query + return s.SelectedNumberChannels > 0 } func (c *ChannelType) doFilter(params []fdsnStationV1Search) bool { @@ -764,10 +750,10 @@ func (c *ChannelType) doFilter(params []fdsnStationV1Search) bool { if p.LocationReg != nil && !matchAnyRegex(c.LocationCode, p.LocationReg) { continue } - if !p.validLatLng(c.Latitude.Value, c.Longitude.Value) { + if !p.validLatLng(c.Latitude, c.Longitude) { continue } - if !p.validBounding(c.Latitude.Value, c.Longitude.Value) { + if !p.validBounding(c.Latitude, c.Longitude) { continue } @@ -827,33 +813,40 @@ func (v fdsnStationV1Search) validStartEnd(start, end time.Time, level int) bool return true } -func (v fdsnStationV1Search) validLatLng(latitude, longitude float64) bool { - if v.MinLatitude != math.MaxFloat64 && latitude < v.MinLatitude { +func (v fdsnStationV1Search) validLatLng(latitude *LatitudeType, longitude *LongitudeType) bool { + if v.MinLatitude != math.MaxFloat64 && (latitude == nil || latitude.Value < v.MinLatitude) { + // request to check latitude: + // 1. this node doesn't have latitude -> check failed + // 2. the value fall outside the range -> check failed + // (similar logics apply for cases below) return false } - if v.MaxLatitude != math.MaxFloat64 && latitude > v.MaxLatitude { + if v.MaxLatitude != math.MaxFloat64 && (latitude == nil || latitude.Value > v.MaxLatitude) { return false } - if v.MinLongitude != math.MaxFloat64 && longitude < v.MinLongitude { + if v.MinLongitude != math.MaxFloat64 && (longitude == nil || longitude.Value < v.MinLongitude) { return false } - if v.MaxLongitude != math.MaxFloat64 && longitude > v.MaxLongitude { + if v.MaxLongitude != math.MaxFloat64 && (longitude == nil || longitude.Value > v.MaxLongitude) { return false } return true } -func (v fdsnStationV1Search) validBounding(latitude, longitude float64) bool { +func (v fdsnStationV1Search) validBounding(latitude *LatitudeType, longitude *LongitudeType) bool { if v.Latitude == math.MaxFloat64 { // not using bounding circle return true } - - d, _, err := wgs84.DistanceBearing(v.Latitude, v.Longitude, latitude, longitude) + if latitude == nil || longitude == nil { + // requested bounding circle, but this node doesn't have lat/lon + return false + } + d, _, err := wgs84.DistanceBearing(v.Latitude, v.Longitude, latitude.Value, longitude.Value) if err != nil { log.Printf("Error checking bounding:%s\n", err.Error()) return false diff --git a/cmd/fdsn-ws/fdsn_station_test.go b/cmd/fdsn-ws/fdsn_station_test.go index 22308c80..e5b9e866 100644 --- a/cmd/fdsn-ws/fdsn_station_test.go +++ b/cmd/fdsn-ws/fdsn_station_test.go @@ -2,10 +2,11 @@ package main import ( "encoding/xml" - wt "github.com/GeoNet/kit/weft/wefttest" "net/url" "strings" "testing" + + wt "github.com/GeoNet/kit/weft/wefttest" ) // NOTE: To run the test, please export : @@ -471,7 +472,50 @@ NZ|ARHZ|-39.263100|176.995900|270.000000|Aropaoanui|2010-03-11T00:00:00|`) format=xml NZ ARA* * EHE* 2001-01-01T00:00:00 * NZ ARH? * EHN* 2001-01-01T00:00:00 *` - testXml := `GeoNetWEL(GNS_Test)Delta2017-09-26T02:37:17New Zealand National Seismograph Network22Private seismograph sitesLocation is given in NZGD2000-38.62769176.12006420Aratiatia Landcorp Farm9 km north of Taupo2007-05-20T23:00:0093Hawke's Bay regional seismic networkLocation is given in WGS84-39.2631176.9959270Aropaoanui28 km north of Napier2010-03-11T00:00:0062` + testXml := ` + + GeoNet + WEL(GNS_Test) + Delta + 2017-09-26T02:37:17 + + New Zealand National Seismograph Network + 2 + 2 + + Private seismograph sites + + Location is given in NZGD2000 + + -38.62769 + 176.12006 + 420 + + Aratiatia Landcorp Farm + 9 km north of Taupo + + 2007-05-20T23:00:00 + 9 + 3 + + + Hawke's Bay regional seismic network + + Location is given in WGS84 + + -39.2631 + 176.9959 + 270 + + Aropaoanui + 28 km north of Napier + + 2010-03-11T00:00:00 + 6 + 2 + + + ` var src FDSNStationXML err = xml.Unmarshal([]byte(testXml), &src) @@ -533,3 +577,208 @@ NZ ARH? * EHN* 2001-01-01T00:00:00 *` } } } + +// test filter. Especially for nodes having no children. +func TestDoFilter(t *testing.T) { + var err error + var fdsn FDSNStationXML + var query url.Values + var hasValue bool + + // + // basic case + // + + // Test network filter - match case + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "STA1", "10", "CHA1") + query.Set("network", "NZ") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if !hasValue { + t.Errorf("expected to have NZ network got empty") + } + + // Test network filter - unmatch case + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "STA1", "10", "CHA1") + query.Set("network", "MC") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if hasValue { + t.Errorf("expected to be empty got %v", fdsn) + } + + // Test station filter - match case + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "STA1", "10", "CHA1") + query.Set("network", "NZ") + query.Set("station", "STA1") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if !hasValue { + t.Errorf("expected to have STA1 station got empty") + } + + // Test station filter - unmatch case + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "STA1", "10", "CHA1") + query.Set("network", "NZ") + query.Set("station", "STA2") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if hasValue { + t.Errorf("expected to be empty got %v", fdsn) + } + + // Test channel filter - match case + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "STA1", "10", "CHA1") + query.Set("network", "NZ") + query.Set("station", "STA1") + query.Set("channel", "CHA1") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if !hasValue { + t.Errorf("expected to have CHA1 channel got empty") + } + + // Test channel filter - unmatch case + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "STA1", "10", "CHA1") + query.Set("network", "NZ") + query.Set("station", "STA1") + query.Set("channel", "CHA2") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if hasValue { + t.Errorf("expected to be empty got %v", fdsn) + } + + // + // complicated cases + // + + // network without stations, asking level station, returns till network level + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "", "", "") + query.Set("network", "NZ") + query.Set("level", "station") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if !hasValue { + t.Errorf("expected to have NZ network got empty") + } + if len(fdsn.Network) != 1 || len(fdsn.Network[0].Station) != 0 { + t.Errorf("exepcted to have NZ network only, got %v", fdsn) + } + + // network without stations, query contains station, should fail + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "", "", "") + query.Set("network", "NZ") + query.Set("network", "STA1") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if hasValue { + t.Errorf("expected to be emptyu got %v", fdsn) + } + + // station without channels, asking level channel, returns till station level + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "STA1", "", "") + query.Set("network", "NZ") + query.Set("station", "STA1") + query.Set("level", "channel") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if !hasValue { + t.Errorf("expected to have STA1 station got empty") + } + if len(fdsn.Network) != 1 || len(fdsn.Network[0].Station) != 1 || len(fdsn.Network[0].Station[0].Channel) != 0 { + t.Errorf("exepcted to have NZ/STA1, got %v", fdsn) + } + + // station without channels, query contains channel, should fail + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "STA1", "", "") + query.Set("network", "NZ") + query.Set("station", "STA1") + query.Set("channel", "CHA1") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if hasValue { + t.Errorf("expected to be emptyu got %v", fdsn) + } + +} + +// helper functions +func testCase(c *FDSNStationXML, query url.Values) (bool, error) { + var e fdsnStationV1Search + var err error + if e, err = parseStationV1(query); err != nil { + return false, err + } + + return c.doFilter([]fdsnStationV1Search{e}), nil +} + +var emptyXsdDatetime = xsdDateTime(emptyDateTime) + +func makeTestFDSN(network, station, location, channel string) FDSNStationXML { + var c = FDSNStationXML{ + Network: []NetworkType{ + { + BaseNodeType: BaseNodeType{ + Code: network, + StartDate: emptyXsdDatetime, + EndDate: emptyXsdDatetime, + }, + Station: []StationType{}, + }, + }, + } + + if station != "" { + c.Network[0].Station = append(c.Network[0].Station, makeTestStation(station)) + + if channel != "" { + c.Network[0].Station[0].Channel = append(c.Network[0].Station[0].Channel, makeTestChannel(channel, location)) + } + } + + return c +} + +func makeTestStation(code string) StationType { + return StationType{ + BaseNodeType: BaseNodeType{ + Code: code, + StartDate: emptyXsdDatetime, + EndDate: emptyXsdDatetime, + }, + Channel: []ChannelType{}, + } +} + +func makeTestChannel(code, locationCode string) ChannelType { + return ChannelType{ + BaseNodeType: BaseNodeType{ + Code: code, + StartDate: emptyXsdDatetime, + EndDate: emptyXsdDatetime, + }, + LocationCode: locationCode, + } +} From 3293efde83104b7b8edbd211b299a394805337a3 Mon Sep 17 00:00:00 2001 From: Howard Wu Date: Mon, 6 Mar 2023 14:06:01 +1300 Subject: [PATCH 2/2] More considerations for station filtering. --- cmd/fdsn-ws/fdsn_station.go | 41 +++++++++++++++++++++++--------- cmd/fdsn-ws/fdsn_station_test.go | 17 +++++++++++++ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/cmd/fdsn-ws/fdsn_station.go b/cmd/fdsn-ws/fdsn_station.go index a73527bf..ab825c94 100644 --- a/cmd/fdsn-ws/fdsn_station.go +++ b/cmd/fdsn-ws/fdsn_station.go @@ -39,6 +39,8 @@ const ( ONBEFOREEND = 0 ONAFTERSTART = 0 AFTER = 1 + + REGEX_ANYTHING = "^.*$" ) var stationAbbreviations = map[string]string{ @@ -663,14 +665,20 @@ func (n *NetworkType) doFilter(params []fdsnStationV1Search) bool { if len(n.Station) == 0 { // for network nodes without children (though unlikely to happen): - // 1. If the query parameter stops at network level, then the match is done - // 2. Otherwise, we're unable to get a match because of empty children (thus returning false) + // If there are query parameters for furthur levels, and there's no "*" (match anything) in the parameters, + // then it'll be impossible to find a match (because we don't have children) for _, p := range matchedParams { - if p.StationReg == nil && p.ChannelReg == nil && p.LocationReg == nil { - return true + if p.StationReg != nil && !contains(p.StationReg, REGEX_ANYTHING) { + return false + } + if p.ChannelReg != nil && !contains(p.ChannelReg, REGEX_ANYTHING) { + return false + } + if p.LocationReg != nil && !contains(p.LocationReg, REGEX_ANYTHING) { + return false } } - return false + return true } for _, s := range n.Station { @@ -715,15 +723,17 @@ func (s *StationType) doFilter(params []fdsnStationV1Search) bool { if len(s.Channel) == 0 { // for station nodes without children: - // 1. If the query parameter stops at station level, then the match is done - // 2. Otherwise, we're unable to get a match (thus returning false) + // If there are query parameters for furthur levels, and there's no "*" (match anything) in the parameters, + // then it'll be impossible to find a match (because we don't have children) for _, p := range matchedParams { - if p.ChannelReg == nil && p.LocationReg == nil { - return true + if p.ChannelReg != nil && !contains(p.ChannelReg, REGEX_ANYTHING) { + return false + } + if p.LocationReg != nil && !contains(p.LocationReg, REGEX_ANYTHING) { + return false } } - - return false + return true } for _, c := range s.Channel { @@ -1007,3 +1017,12 @@ func (d xsdDateTime) MarshalFormatText() string { b, _ := d.MarshalText() return string(b) } + +func contains(slice []string, value string) bool { + for _, s := range slice { + if s == value { + return true + } + } + return false +} diff --git a/cmd/fdsn-ws/fdsn_station_test.go b/cmd/fdsn-ws/fdsn_station_test.go index e5b9e866..1b36a502 100644 --- a/cmd/fdsn-ws/fdsn_station_test.go +++ b/cmd/fdsn-ws/fdsn_station_test.go @@ -721,6 +721,23 @@ func TestDoFilter(t *testing.T) { t.Errorf("expected to be emptyu got %v", fdsn) } + // two station without children, we should see both + query = make(map[string][]string) + fdsn = makeTestFDSN("NZ", "STA1", "", "") + fdsn.Network[0].Station = append(fdsn.Network[0].Station, makeTestStation("STA2")) + query.Set("network", "NZ") + query.Set("station", "STA1,STA2") + query.Set("channel", "*") + if hasValue, err = testCase(&fdsn, query); err != nil { + t.Error(err) + } + if !hasValue { + t.Errorf("expected to be values got empty result") + } + // we should get 2 results + if len(fdsn.Network[0].Station) != 2 { + t.Errorf("expected to be 2 stations got %v", fdsn) + } } // helper functions