diff --git a/CHANGELOG.md b/CHANGELOG.md index b48633a68e..513ff49193 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,10 @@ This is intended as an overview of the major changes v5.21-SNAPSHOT === - Analyzer: + - Improve the Calculator setup. +- New/improved detections - Tests: Edge on Linux + - Handle Facebook agents with lists of keyvalues v5.20 === diff --git a/analyzer/src/main/antlr4/nl/basjes/parse/useragent/parser/UserAgent.g4 b/analyzer/src/main/antlr4/nl/basjes/parse/useragent/parser/UserAgent.g4 index 63c472e6c1..ae312d1ae3 100644 --- a/analyzer/src/main/antlr4/nl/basjes/parse/useragent/parser/UserAgent.g4 +++ b/analyzer/src/main/antlr4/nl/basjes/parse/useragent/parser/UserAgent.g4 @@ -384,6 +384,8 @@ productVersionWithCommas | base64 | singleVersionWithCommas | SPECIALVERSIONWORDS + // This next one only occurs in a Facebook useragent + | CURLYBRACEOPEN keyValue ( COMMA keyValue )* CURLYBRACECLOSE ; diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/AbstractUserAgentAnalyzerDirect.java b/analyzer/src/main/java/nl/basjes/parse/useragent/AbstractUserAgentAnalyzerDirect.java index 17a136034d..cc1a670a97 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/AbstractUserAgentAnalyzerDirect.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/AbstractUserAgentAnalyzerDirect.java @@ -71,6 +71,7 @@ import nl.basjes.parse.useragent.analyze.treewalker.steps.walk.StepPrev; import nl.basjes.parse.useragent.analyze.treewalker.steps.walk.StepPrevN; import nl.basjes.parse.useragent.analyze.treewalker.steps.walk.StepUp; +import nl.basjes.parse.useragent.calculate.CalculateAgentClass; import nl.basjes.parse.useragent.calculate.CalculateAgentEmail; import nl.basjes.parse.useragent.calculate.CalculateAgentName; import nl.basjes.parse.useragent.calculate.CalculateDeviceBrand; @@ -115,7 +116,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static nl.basjes.parse.useragent.UserAgent.AGENT_CLASS; -import static nl.basjes.parse.useragent.UserAgent.AGENT_INFORMATION_EMAIL; import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME; import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME_VERSION; import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME_VERSION_MAJOR; @@ -134,7 +134,6 @@ import static nl.basjes.parse.useragent.UserAgent.LAYOUT_ENGINE_VERSION; import static nl.basjes.parse.useragent.UserAgent.LAYOUT_ENGINE_VERSION_MAJOR; import static nl.basjes.parse.useragent.UserAgent.MutableUserAgent.isSystemField; -import static nl.basjes.parse.useragent.UserAgent.NETWORK_TYPE; import static nl.basjes.parse.useragent.UserAgent.OPERATING_SYSTEM_CLASS; import static nl.basjes.parse.useragent.UserAgent.OPERATING_SYSTEM_NAME; import static nl.basjes.parse.useragent.UserAgent.OPERATING_SYSTEM_NAME_VERSION; @@ -283,6 +282,7 @@ public static void configureKryo(Object kryoInstance) { kryo.register(CalculateAgentEmail.class); kryo.register(CalculateAgentName.class); + kryo.register(CalculateAgentClass.class); kryo.register(CalculateDeviceBrand.class); kryo.register(CalculateDeviceName.class); kryo.register(CalculateNetworkType.class); @@ -1627,28 +1627,35 @@ public B immediateInitialization() { return (B)this; } - private void addCalculator(FieldCalculator calculator) { - fieldCalculators.add(calculator); - if (uaa.wantedFieldNames != null) { - Collections.addAll(uaa.wantedFieldNames, calculator.getDependencies()); - } - } + protected Set allFieldsForWhichACalculatorExists = new HashSet<>(); - private void addCalculatedMajorVersionField(String result, String dependency) { - if (uaa.isWantedField(result)) { - fieldCalculators.add(new MajorVersionCalculator(result, dependency)); + private void registerFieldCalculator(FieldCalculator fieldCalculator) { + String calculatedFieldName = fieldCalculator.getCalculatedFieldName(); + allFieldsForWhichACalculatorExists.add(calculatedFieldName); + if (uaa.isWantedField(calculatedFieldName)) { + fieldCalculators.add(fieldCalculator); if (uaa.wantedFieldNames != null) { - Collections.addAll(uaa.wantedFieldNames, dependency); + uaa.wantedFieldNames.addAll(fieldCalculator.getDependencies()); } } } - private void addCalculatedConcatNONDuplicated(String result, String first, String second) { - if (uaa.isWantedField(result)) { - fieldCalculators.add(new ConcatNONDuplicatedCalculator(result, first, second)); - if (uaa.wantedFieldNames != null) { - Collections.addAll(uaa.wantedFieldNames, first, second); + protected void verifyCalculatorDependencyOrdering() { + // Verify calculator dependencies ordering + Set seenCalculatedFields = new HashSet<>(); + for (FieldCalculator fieldCalculator: fieldCalculators) { + for (String dependency: fieldCalculator.getDependencies()){ + if (allFieldsForWhichACalculatorExists.contains(dependency)) { + if (!seenCalculatedFields.contains(dependency)) { + throw new InvalidParserConfigurationException( + "Calculator ordering is wrong:" + + "For " + fieldCalculator.getCalculatedFieldName() + + " we need " + dependency + " which is a " + + "calculated field but it has not yet been calculated."); + } + } } + seenCalculatedFields.add(fieldCalculator.getCalculatedFieldName()); } } @@ -1669,40 +1676,27 @@ public UAA build() { uaa.wantedFieldNames.add(DEVICE_CLASS); } - addCalculatedConcatNONDuplicated(AGENT_NAME_VERSION_MAJOR, AGENT_NAME, AGENT_VERSION_MAJOR); - addCalculatedConcatNONDuplicated(AGENT_NAME_VERSION, AGENT_NAME, AGENT_VERSION); - addCalculatedMajorVersionField(AGENT_VERSION_MAJOR, AGENT_VERSION); - - addCalculatedConcatNONDuplicated(WEBVIEW_APP_NAME_VERSION_MAJOR, WEBVIEW_APP_NAME, WEBVIEW_APP_VERSION_MAJOR); - addCalculatedMajorVersionField(WEBVIEW_APP_VERSION_MAJOR, WEBVIEW_APP_VERSION); + registerFieldCalculator(new ConcatNONDuplicatedCalculator(AGENT_NAME_VERSION_MAJOR, AGENT_NAME, AGENT_VERSION_MAJOR)); + registerFieldCalculator(new ConcatNONDuplicatedCalculator(AGENT_NAME_VERSION, AGENT_NAME, AGENT_VERSION)); + registerFieldCalculator(new MajorVersionCalculator(AGENT_VERSION_MAJOR, AGENT_VERSION)); - addCalculatedConcatNONDuplicated(LAYOUT_ENGINE_NAME_VERSION_MAJOR, LAYOUT_ENGINE_NAME, LAYOUT_ENGINE_VERSION_MAJOR); - addCalculatedConcatNONDuplicated(LAYOUT_ENGINE_NAME_VERSION, LAYOUT_ENGINE_NAME, LAYOUT_ENGINE_VERSION); - addCalculatedMajorVersionField(LAYOUT_ENGINE_VERSION_MAJOR, LAYOUT_ENGINE_VERSION); + registerFieldCalculator(new ConcatNONDuplicatedCalculator(WEBVIEW_APP_NAME_VERSION_MAJOR, WEBVIEW_APP_NAME, WEBVIEW_APP_VERSION_MAJOR)); + registerFieldCalculator(new MajorVersionCalculator(WEBVIEW_APP_VERSION_MAJOR, WEBVIEW_APP_VERSION)); - addCalculatedMajorVersionField(OPERATING_SYSTEM_NAME_VERSION_MAJOR, OPERATING_SYSTEM_NAME_VERSION); - addCalculatedConcatNONDuplicated(OPERATING_SYSTEM_NAME_VERSION, OPERATING_SYSTEM_NAME, OPERATING_SYSTEM_VERSION); - addCalculatedMajorVersionField(OPERATING_SYSTEM_VERSION_MAJOR, OPERATING_SYSTEM_VERSION); + registerFieldCalculator(new ConcatNONDuplicatedCalculator(LAYOUT_ENGINE_NAME_VERSION_MAJOR, LAYOUT_ENGINE_NAME, LAYOUT_ENGINE_VERSION_MAJOR)); + registerFieldCalculator(new ConcatNONDuplicatedCalculator(LAYOUT_ENGINE_NAME_VERSION, LAYOUT_ENGINE_NAME, LAYOUT_ENGINE_VERSION)); + registerFieldCalculator(new MajorVersionCalculator(LAYOUT_ENGINE_VERSION_MAJOR, LAYOUT_ENGINE_VERSION)); - if (uaa.isWantedField(AGENT_NAME)) { - addCalculator(new CalculateAgentName()); - } - - if (uaa.isWantedField(NETWORK_TYPE)) { - addCalculator(new CalculateNetworkType()); - } - - if (uaa.isWantedField(DEVICE_NAME)) { - addCalculator(new CalculateDeviceName()); - } - - if (uaa.isWantedField(DEVICE_BRAND)) { - addCalculator(new CalculateDeviceBrand()); - } + registerFieldCalculator(new MajorVersionCalculator(OPERATING_SYSTEM_NAME_VERSION_MAJOR, OPERATING_SYSTEM_NAME_VERSION)); + registerFieldCalculator(new ConcatNONDuplicatedCalculator(OPERATING_SYSTEM_NAME_VERSION, OPERATING_SYSTEM_NAME, OPERATING_SYSTEM_VERSION)); + registerFieldCalculator(new MajorVersionCalculator(OPERATING_SYSTEM_VERSION_MAJOR, OPERATING_SYSTEM_VERSION)); - if (uaa.isWantedField(AGENT_INFORMATION_EMAIL)) { - addCalculator(new CalculateAgentEmail()); - } + registerFieldCalculator(new CalculateAgentClass()); + registerFieldCalculator(new CalculateAgentName()); + registerFieldCalculator(new CalculateNetworkType()); + registerFieldCalculator(new CalculateDeviceName()); + registerFieldCalculator(new CalculateDeviceBrand()); + registerFieldCalculator(new CalculateAgentEmail()); Collections.reverse(fieldCalculators); uaa.setFieldCalculators(fieldCalculators); diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentClass.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentClass.java new file mode 100644 index 0000000000..911cf9d729 --- /dev/null +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentClass.java @@ -0,0 +1,54 @@ +/* + * Yet Another UserAgent Analyzer + * Copyright (C) 2013-2020 Niels Basjes + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nl.basjes.parse.useragent.calculate; + +import nl.basjes.parse.useragent.AgentField; +import nl.basjes.parse.useragent.UserAgent.MutableUserAgent; + +import java.util.Collections; +import java.util.Set; + +import static nl.basjes.parse.useragent.UserAgent.AGENT_CLASS; +import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME; + +public class CalculateAgentClass extends FieldCalculator { + @Override + public void calculate(MutableUserAgent userAgent) { + // Cleanup the class of the useragent + AgentField agentClass = userAgent.get(AGENT_CLASS); + if (agentClass.isDefaultValue()) { + AgentField agentName = userAgent.get(AGENT_NAME); + if (!agentName.isDefaultValue()) { + userAgent.setForced( + AGENT_CLASS, + "Special", + 1); + } + } + } + + @Override + public String getCalculatedFieldName() { + return AGENT_CLASS; + } + + @Override + public Set getDependencies() { + return Collections.singleton(AGENT_NAME); + } +} diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentEmail.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentEmail.java index 254adc9137..2927a88cb5 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentEmail.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentEmail.java @@ -23,7 +23,7 @@ import static nl.basjes.parse.useragent.UserAgent.AGENT_INFORMATION_EMAIL; -public class CalculateAgentEmail implements FieldCalculator { +public class CalculateAgentEmail extends FieldCalculator { @Override public void calculate(MutableUserAgent userAgent) { // The email address is a mess @@ -37,8 +37,7 @@ public void calculate(MutableUserAgent userAgent) { } @Override - public String toString() { - return "Calculate " + AGENT_INFORMATION_EMAIL; + public String getCalculatedFieldName() { + return AGENT_INFORMATION_EMAIL; } - } diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentName.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentName.java index 760b873377..4f9be0d4ba 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentName.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentName.java @@ -21,12 +21,15 @@ import nl.basjes.parse.useragent.UserAgent.MutableUserAgent; import nl.basjes.parse.useragent.utils.Normalize; +import java.util.Collections; +import java.util.Set; + import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME; import static nl.basjes.parse.useragent.UserAgent.DEVICE_BRAND; import static nl.basjes.parse.useragent.UserAgent.NULL_VALUE; import static nl.basjes.parse.useragent.utils.Normalize.isLowerCase; -public class CalculateAgentName implements FieldCalculator { +public class CalculateAgentName extends FieldCalculator { @Override public void calculate(MutableUserAgent userAgent) { // Cleanup the name of the useragent @@ -61,13 +64,12 @@ public void calculate(MutableUserAgent userAgent) { } @Override - public String[] getDependencies() { - return new String[]{DEVICE_BRAND}; + public String getCalculatedFieldName() { + return AGENT_NAME; } @Override - public String toString() { - return "Calculate " + AGENT_NAME; + public Set getDependencies() { + return Collections.singleton(DEVICE_BRAND); } - } diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceBrand.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceBrand.java index 4b59086842..0721e5a981 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceBrand.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceBrand.java @@ -35,7 +35,7 @@ import static nl.basjes.parse.useragent.utils.HostnameExtracter.extractHostname; import static org.apache.hc.client5.http.psl.DomainType.ICANN; -public class CalculateDeviceBrand implements FieldCalculator { +public class CalculateDeviceBrand extends FieldCalculator { private final Set unwantedUrlBrands; private final Set unwantedEmailBrands; @@ -168,13 +168,12 @@ private String extractCompanyFromHostName(String hostname, Set blackList } @Override - public String[] getDependencies() { - return new String[]{AGENT_INFORMATION_URL, AGENT_INFORMATION_EMAIL}; + public String getCalculatedFieldName() { + return DEVICE_BRAND; } @Override - public String toString() { - return "Calculate " + DEVICE_BRAND; + public Set getDependencies() { + return new HashSet<>(Arrays.asList(AGENT_INFORMATION_URL, AGENT_INFORMATION_EMAIL)); } - } diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceName.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceName.java index 79d448a6b5..2cd4bcd8dd 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceName.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceName.java @@ -21,6 +21,8 @@ import nl.basjes.parse.useragent.UserAgent.MutableUserAgent; import nl.basjes.parse.useragent.utils.Normalize; +import java.util.Collections; +import java.util.Set; import java.util.regex.Pattern; import static java.util.regex.Pattern.CASE_INSENSITIVE; @@ -29,7 +31,7 @@ import static nl.basjes.parse.useragent.UserAgent.DEVICE_NAME; import static nl.basjes.parse.useragent.UserAgent.UNKNOWN_VALUE; -public class CalculateDeviceName implements FieldCalculator { +public class CalculateDeviceName extends FieldCalculator { private static final Pattern CLEAN_1_PATTERN = Pattern.compile("AppleWebKit", CASE_INSENSITIVE | LITERAL); @@ -64,8 +66,13 @@ public void calculate(MutableUserAgent userAgent) { } @Override - public String[] getDependencies() { - return new String[]{DEVICE_BRAND}; + public String getCalculatedFieldName() { + return DEVICE_NAME; + } + + @Override + public Set getDependencies() { + return Collections.singleton(DEVICE_BRAND); } @Override diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateNetworkType.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateNetworkType.java index 38dc45287b..237d61a069 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateNetworkType.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateNetworkType.java @@ -23,7 +23,7 @@ import static nl.basjes.parse.useragent.UserAgent.NETWORK_TYPE; -public class CalculateNetworkType implements FieldCalculator { +public class CalculateNetworkType extends FieldCalculator { @Override public void calculate(MutableUserAgent userAgent) { @@ -37,6 +37,11 @@ public void calculate(MutableUserAgent userAgent) { } } + @Override + public String getCalculatedFieldName() { + return NETWORK_TYPE; + } + @Override public String toString() { return "Calculate " + NETWORK_TYPE; diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/ConcatNONDuplicatedCalculator.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/ConcatNONDuplicatedCalculator.java index 848084404d..8e1dd30294 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/ConcatNONDuplicatedCalculator.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/ConcatNONDuplicatedCalculator.java @@ -20,9 +20,13 @@ import nl.basjes.parse.useragent.AgentField; import nl.basjes.parse.useragent.UserAgent.MutableUserAgent; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + import static nl.basjes.parse.useragent.UserAgent.NULL_VALUE; -public class ConcatNONDuplicatedCalculator implements FieldCalculator { +public class ConcatNONDuplicatedCalculator extends FieldCalculator { private final String targetName; private final String firstName; @@ -78,8 +82,13 @@ public void calculate(MutableUserAgent userAgent) { } @Override - public String[] getDependencies() { - return new String[]{firstName, secondName}; + public String getCalculatedFieldName() { + return targetName; + } + + @Override + public Set getDependencies() { + return new HashSet<>(Arrays.asList(firstName, secondName)); } @Override diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/FieldCalculator.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/FieldCalculator.java index 5370c1a649..a40426bdb8 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/FieldCalculator.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/FieldCalculator.java @@ -20,12 +20,20 @@ import nl.basjes.parse.useragent.UserAgent.MutableUserAgent; import java.io.Serializable; +import java.util.Collections; +import java.util.Set; -public interface FieldCalculator extends Serializable { - void calculate(MutableUserAgent userAgent); +public abstract class FieldCalculator implements Serializable { + public abstract void calculate(MutableUserAgent userAgent); - default String[] getDependencies() { - return new String[0]; + public abstract String getCalculatedFieldName(); + + public Set getDependencies() { + return Collections.emptySet(); } + @Override + public String toString() { + return "Calculate " + getDependencies() + " ==> " + getCalculatedFieldName(); + } } diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/MajorVersionCalculator.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/MajorVersionCalculator.java index 60cebb2688..6ea6d7ed9d 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/MajorVersionCalculator.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/MajorVersionCalculator.java @@ -21,9 +21,12 @@ import nl.basjes.parse.useragent.UserAgent.MutableUserAgent; import nl.basjes.parse.useragent.utils.VersionSplitter; +import java.util.Collections; +import java.util.Set; + import static nl.basjes.parse.useragent.UserAgent.NULL_VALUE; -public class MajorVersionCalculator implements FieldCalculator { +public class MajorVersionCalculator extends FieldCalculator { private String versionName; private String majorVersionName; @@ -53,8 +56,13 @@ public void calculate(MutableUserAgent userAgent) { } @Override - public String[] getDependencies() { - return new String[]{versionName}; + public String getCalculatedFieldName() { + return majorVersionName; + } + + @Override + public Set getDependencies() { + return Collections.singleton(versionName); } @Override diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/debug/AbstractUserAgentAnalyzerTester.java b/analyzer/src/main/java/nl/basjes/parse/useragent/debug/AbstractUserAgentAnalyzerTester.java index 35719e9ce1..63e29727e5 100644 --- a/analyzer/src/main/java/nl/basjes/parse/useragent/debug/AbstractUserAgentAnalyzerTester.java +++ b/analyzer/src/main/java/nl/basjes/parse/useragent/debug/AbstractUserAgentAnalyzerTester.java @@ -583,7 +583,9 @@ public abstract static class AbstractUserAgentAnalyzerTesterBuilder>>"]' - 'AgentClass : 1001 :"Browser Webview"' - - 'AgentName : 1001 :LookUp[FaceBookWebviewNames;@FBANVersion]' + - 'AgentName : 1001 :LookUp[FaceBookWebviewNames;@FBAN]' - 'AgentVersion : 11 :"??"' - - 'WebviewAppName : 1001 :LookUp[FaceBookAppNames;@FBANVersion]' + - 'WebviewAppName : 1001 :LookUp[FaceBookAppNames;@FBAN]' - 'WebviewAppVersion : 1001 :"??"' - matcher: @@ -64,6 +80,8 @@ config: - 'FBAN :agent.product.(1-2)comments.entry.(1)product.name="FBAN"^.version' - 'FBAV :agent.product.(1-2)comments.entry.(1)product.name="FBAV"^.version' extract: + - 'DeviceClass : 50 :"Mobile"' + - 'OperatingSystemName : 1 : LookUp[FaceBookAppOSNames;@FBAN;"<<>>"]' - 'AgentClass : 1001 :"Browser Webview"' - 'AgentName : 1001 :LookUp[FaceBookWebviewNames;@FBAN]' - 'AgentVersion : 11 :"??"' @@ -75,10 +93,11 @@ config: - 'FBAN :agent.product.(1-2)comments.entry.(1)product.name="FBAN"^.version' - 'FBAV :agent.product.(1-2)comments.entry.(1)product.name="FBAV"^.version' extract: + - 'DeviceClass : 50 :"Mobile"' - 'AgentClass : 1000 :"Mobile App"' - - 'AgentName : 1000 :@FBAN' - - 'AgentVersion : 5 :@FBAV' - - 'WebviewAppName : 1000 :@FBAN' + - 'AgentName : 1000 :Concat["Facebook ";@FBAN]' + - 'AgentVersion : 9 :@FBAV' + - 'WebviewAppName : 1000 :Concat["Facebook ";@FBAN]' - 'WebviewAppVersion : 1000 :@FBAV' - matcher: @@ -98,9 +117,9 @@ config: - 'FBAV :agent.product.(1-2)comments.entry.(1)product.name="FBAV"^.version' extract: - 'AgentClass : 1000 :"Mobile App"' - - 'AgentName : 1000 :@FBIAB' + - 'AgentName : 1000 :Concat["Facebook ";@FBIAB]' - 'AgentVersion : 5 :@FBAV' - - 'WebviewAppName : 1000 :@FBIAB' + - 'WebviewAppName : 1000 :Concat["Facebook ";@FBIAB]' - 'WebviewAppVersion : 1000 :@FBAV' - matcher: @@ -156,6 +175,14 @@ config: - 'DeviceName : 201 :LookUp[AppleDeviceName;@FBMD]' - 'DeviceVersion : 201 :LookUp[AppleDeviceVersion;@FBDV]' +- matcher: + variable: + - 'FBBD :agent.product.(1-2)comments.entry.(1)product.name="FBBD"^.version' + - 'FBDV :agent.product.(1-2)comments.entry.(1)product.name="FBDV"^.version' + extract: + - 'DeviceBrand : 201 :@FBBD' + - 'DeviceName : 201 :@FBDV' + - matcher: extract: - 'FacebookOperatingSystemName : 3000 :agent.product.(1-2)comments.entry.(1)product.name="FBSN"^.version' @@ -168,6 +195,12 @@ config: - 'OperatingSystemName : 199 :agent.product.(1-2)comments.entry.(1)product.name="FBSN"^.version' - 'OperatingSystemVersion : 199 :agent.product.(1-2)comments.entry.(1)product.name="FBSV"^.version' +- matcher: + extract: + - 'OperatingSystemClass : 9 :"Mobile"' + - 'OperatingSystemVersion : 199 :agent.product.(1-2)comments.entry.(1)product.name="FBSV"^.version' + + - lookup: name: 'OSNameCleanup' map: @@ -440,12 +473,12 @@ config: OperatingSystemVersionMajor : '8' OperatingSystemNameVersion : 'iOS 8.0.2' OperatingSystemNameVersionMajor : 'iOS 8' - LayoutEngineClass : 'Browser' - LayoutEngineName : 'Mozilla' - LayoutEngineVersion : '5.0' - LayoutEngineVersionMajor : '5' - LayoutEngineNameVersion : 'Mozilla 5.0' - LayoutEngineNameVersionMajor : 'Mozilla 5' + LayoutEngineClass : 'Unknown' + LayoutEngineName : 'Unknown' + LayoutEngineVersion : '??' + LayoutEngineVersionMajor : '??' + LayoutEngineNameVersion : 'Unknown ??' + LayoutEngineNameVersionMajor : 'Unknown ??' AgentClass : 'Browser Webview' AgentName : 'UIWebView' AgentVersion : '??' @@ -775,3 +808,217 @@ config: FacebookOperatingSystemName : 'iOS' FacebookOperatingSystemVersion : '13.1.3' Carrier : 'EE' + +- test: + input: + user_agent_string: '[FBAN/FB4A;FBAV/266.0.0.64.124;FBBV/209629367;FBDM/{density=3.375,width=1080,height=2139};FBLC/fr_FR;FBRV/0;FBCR/VOO;FBMF/HUAWEI;FBBD/HUAWEI;FBPN/com.facebook.katana;FBDV/POT-LX1;FBSV/9;FBOP/1;FBCA/arm64-v8a:;]' + expected: + DeviceClass : 'Mobile' + DeviceName : 'Huawei POT-LX1' + DeviceBrand : 'Huawei' + DeviceCpu : 'ARM64' + DeviceCpuBits : '64' + OperatingSystemClass : 'Mobile' + OperatingSystemName : 'Android' + OperatingSystemVersion : '9' + OperatingSystemVersionMajor : '9' + OperatingSystemNameVersion : 'Android 9' + OperatingSystemNameVersionMajor : 'Android 9' + LayoutEngineClass : 'Unknown' + LayoutEngineName : 'Unknown' + LayoutEngineVersion : '??' + LayoutEngineVersionMajor : '??' + LayoutEngineNameVersion : 'Unknown ??' + LayoutEngineNameVersionMajor : 'Unknown ??' + AgentClass : 'Browser Webview' + AgentName : 'Chrome Webview' + AgentVersion : '??' + AgentVersionMajor : '??' + AgentNameVersion : 'Chrome Webview ??' + AgentNameVersionMajor : 'Chrome Webview ??' + AgentBuild : '209629367' + AgentLanguage : 'French (France)' + AgentLanguageCode : 'fr-fr' + WebviewAppName : 'Facebook App for Android' + WebviewAppVersion : '266.0.0.64.124' + WebviewAppVersionMajor : '266' + WebviewAppNameVersionMajor : 'Facebook App for Android 266' + FacebookCarrier : 'VOO' + FacebookFBOP : '1' + Carrier : 'VOO' + +- test: + input: + user_agent_string: '[FBAN/FB4A;FBAV/288.1.0.47.123;FBBV/245303567;FBDM/{density=1.75,width=720,height=1382};FBLC/sq_AL;FBRV/0;FBCR/vodafone GR;FBMF/samsung;FBBD/samsung;FBPN/com.facebook.katana;FBDV/SM-A105FN;FBSV/10;FBOP/19;FBCA/armeabi-v7a:armeabi;]' + expected: + DeviceClass : 'Mobile' + DeviceName : 'Samsung SM-A105FN' + DeviceBrand : 'Samsung' + OperatingSystemClass : 'Mobile' + OperatingSystemName : 'Android' + OperatingSystemVersion : '10' + OperatingSystemVersionMajor : '10' + OperatingSystemNameVersion : 'Android 10' + OperatingSystemNameVersionMajor : 'Android 10' + LayoutEngineClass : 'Unknown' + LayoutEngineName : 'Unknown' + LayoutEngineVersion : '??' + LayoutEngineVersionMajor : '??' + LayoutEngineNameVersion : 'Unknown ??' + LayoutEngineNameVersionMajor : 'Unknown ??' + AgentClass : 'Browser Webview' + AgentName : 'Chrome Webview' + AgentVersion : '??' + AgentVersionMajor : '??' + AgentNameVersion : 'Chrome Webview ??' + AgentNameVersionMajor : 'Chrome Webview ??' + AgentBuild : '245303567' + AgentLanguage : 'Albanian (Albania)' + AgentLanguageCode : 'sq-al' + WebviewAppName : 'Facebook App for Android' + WebviewAppVersion : '288.1.0.47.123' + WebviewAppVersionMajor : '288' + WebviewAppNameVersionMajor : 'Facebook App for Android 288' + FacebookCarrier : 'vodafone GR' + FacebookFBOP : '19' + Carrier : 'vodafone GR' + + +- test: + input: + user_agent_string: '[FBAN/FB4A;FBAV/289.0.0.40.121;FBBV/246888191;FBDM/{density=2.625,width=1080,height=2047};FBLC/tr_TR;FBRV/248143325;FBCR/vodafone NL;FBMF/samsung;FBBD/samsung;FBPN/com.facebook.katana;FBDV/SM-G975F;FBSV/10;FBOP/19;FBCA/arm64-v8a:;]' + expected: + DeviceClass : 'Mobile' + DeviceName : 'Samsung SM-G975F' + DeviceBrand : 'Samsung' + DeviceCpu : 'ARM64' + DeviceCpuBits : '64' + OperatingSystemClass : 'Mobile' + OperatingSystemName : 'Android' + OperatingSystemVersion : '10' + OperatingSystemVersionMajor : '10' + OperatingSystemNameVersion : 'Android 10' + OperatingSystemNameVersionMajor : 'Android 10' + LayoutEngineClass : 'Unknown' + LayoutEngineName : 'Unknown' + LayoutEngineVersion : '??' + LayoutEngineVersionMajor : '??' + LayoutEngineNameVersion : 'Unknown ??' + LayoutEngineNameVersionMajor : 'Unknown ??' + AgentClass : 'Browser Webview' + AgentName : 'Chrome Webview' + AgentVersion : '??' + AgentVersionMajor : '??' + AgentNameVersion : 'Chrome Webview ??' + AgentNameVersionMajor : 'Chrome Webview ??' + AgentBuild : '246888191' + AgentLanguage : 'Turkish (Turkey)' + AgentLanguageCode : 'tr-tr' + WebviewAppName : 'Facebook App for Android' + WebviewAppVersion : '289.0.0.40.121' + WebviewAppVersionMajor : '289' + WebviewAppNameVersionMajor : 'Facebook App for Android 289' + FacebookCarrier : 'vodafone NL' + FacebookFBOP : '19' + Carrier : 'vodafone NL' + + +- test: + input: + user_agent_string: 'Mozilla/5.0 (Linux; Android 10; POT-LX1 Build/HUAWEIPOT-L21; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/85.0.4183.127 Mobile Safari/537.36 [FBAN/EMA;FBLC/nl_NL;FBAV/218.0.0.6.119;FBDM/DisplayMetrics{density=3.0, width=1080, height=2139, scaledDensity=3.0, xdpi=403.411, ydpi=407.095, densityDpi=480, noncompatWidthPixels=1080, noncompatHeightPixels=2139, noncompatDensity=3.0, noncompatDensityDpi=480, noncompatXdpi=403.411, noncompatYdpi=407.095};]' + expected: + DeviceClass : 'Phone' + DeviceName : 'Huawei POT-LX1' + DeviceBrand : 'Huawei' + OperatingSystemClass : 'Mobile' + OperatingSystemName : 'Android' + OperatingSystemVersion : '10' + OperatingSystemVersionMajor : '10' + OperatingSystemNameVersion : 'Android 10' + OperatingSystemNameVersionMajor : 'Android 10' + OperatingSystemVersionBuild : 'HUAWEIPOT-L21' + LayoutEngineClass : 'Browser' + LayoutEngineName : 'Blink' + LayoutEngineVersion : '85.0' + LayoutEngineVersionMajor : '85' + LayoutEngineNameVersion : 'Blink 85.0' + LayoutEngineNameVersionMajor : 'Blink 85' + AgentClass : 'Browser Webview' + AgentName : 'Chrome Webview' + AgentVersion : '85.0.4183.127' + AgentVersionMajor : '85' + AgentNameVersion : 'Chrome Webview 85.0.4183.127' + AgentNameVersionMajor : 'Chrome Webview 85' + AgentLanguage : 'Dutch (Netherlands)' + AgentLanguageCode : 'nl-nl' + WebviewAppName : 'Facebook EMA' + WebviewAppVersion : '218.0.0.6.119' + WebviewAppVersionMajor : '218' + WebviewAppNameVersionMajor : 'Facebook EMA 218' + + +- test: + input: + user_agent_string: 'Mozilla/5.0 (Linux; Android 9; SM-J330FN Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/85.0.4183.127 Mobile Safari/537.36 [FBAN/EMA;FBLC/nl_NL;FBAV/218.0.0.6.119;FBDM/DisplayMetrics{density=2.0, width=720, height=1280, scaledDensity=2.0, xdpi=294.967, ydpi=295.563};]' + expected: + DeviceClass : 'Phone' + DeviceName : 'Samsung SM-J330FN' + DeviceBrand : 'Samsung' + OperatingSystemClass : 'Mobile' + OperatingSystemName : 'Android' + OperatingSystemVersion : '9' + OperatingSystemVersionMajor : '9' + OperatingSystemNameVersion : 'Android 9' + OperatingSystemNameVersionMajor : 'Android 9' + OperatingSystemVersionBuild : 'PPR1.180610.011' + LayoutEngineClass : 'Browser' + LayoutEngineName : 'Blink' + LayoutEngineVersion : '85.0' + LayoutEngineVersionMajor : '85' + LayoutEngineNameVersion : 'Blink 85.0' + LayoutEngineNameVersionMajor : 'Blink 85' + AgentClass : 'Browser Webview' + AgentName : 'Chrome Webview' + AgentVersion : '85.0.4183.127' + AgentVersionMajor : '85' + AgentNameVersion : 'Chrome Webview 85.0.4183.127' + AgentNameVersionMajor : 'Chrome Webview 85' + AgentLanguage : 'Dutch (Netherlands)' + AgentLanguageCode : 'nl-nl' + WebviewAppName : 'Facebook EMA' + WebviewAppVersion : '218.0.0.6.119' + WebviewAppVersionMajor : '218' + WebviewAppNameVersionMajor : 'Facebook EMA 218' + +- test: + input: + user_agent_string: 'Mozilla/5.0 (Linux; Android 10; Nokia 8.1 Build/QKQ1.190828.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/85.0.4183.127 Mobile Safari/537.36 [FBAN/EMA;FBLC/nl_NL;FBAV/218.0.0.6.119;FBDM/DisplayMetrics{density=2.625, width=1080, height=2034, scaledDensity=2.625, xdpi=403.411, ydpi=401.749};]' + expected: + DeviceClass : 'Phone' + DeviceName : 'Nokia 8.1' + DeviceBrand : 'Nokia' + OperatingSystemClass : 'Mobile' + OperatingSystemName : 'Android' + OperatingSystemVersion : '10' + OperatingSystemVersionMajor : '10' + OperatingSystemNameVersion : 'Android 10' + OperatingSystemNameVersionMajor : 'Android 10' + OperatingSystemVersionBuild : 'QKQ1.190828.002' + LayoutEngineClass : 'Browser' + LayoutEngineName : 'Blink' + LayoutEngineVersion : '85.0' + LayoutEngineVersionMajor : '85' + LayoutEngineNameVersion : 'Blink 85.0' + LayoutEngineNameVersionMajor : 'Blink 85' + AgentClass : 'Browser Webview' + AgentName : 'Chrome Webview' + AgentVersion : '85.0.4183.127' + AgentVersionMajor : '85' + AgentNameVersion : 'Chrome Webview 85.0.4183.127' + AgentNameVersionMajor : 'Chrome Webview 85' + AgentLanguage : 'Dutch (Netherlands)' + AgentLanguageCode : 'nl-nl' + WebviewAppName : 'Facebook EMA' + WebviewAppVersion : '218.0.0.6.119' + WebviewAppVersionMajor : '218' + WebviewAppNameVersionMajor : 'Facebook EMA 218' diff --git a/analyzer/src/main/resources/UserAgents/Robots.yaml b/analyzer/src/main/resources/UserAgents/Robots.yaml index 261bfbfec8..2c61326c54 100644 --- a/analyzer/src/main/resources/UserAgents/Robots.yaml +++ b/analyzer/src/main/resources/UserAgents/Robots.yaml @@ -78,6 +78,7 @@ config: 'lwp-request' : 'lwp-request|lwp-request' 'libwww-perl' : 'libwww-perl|libwww-perl' 'spray-can' : 'spray-can|spray-can' + 'facebookcatalog' : 'Facebook|Facebook Catalog' - lookup: name: 'NamedSpecialLayoutEngines' @@ -9352,3 +9353,30 @@ config: AgentNameVersion : 'Acme.sh 2.8.6' AgentNameVersionMajor : 'Acme.sh 2' AgentInformationUrl : 'https://github.com/acmesh-official/acme.sh' + + +- test: + input: + user_agent_string: 'facebookcatalog/1.0' + expected: + DeviceClass : 'Robot' # + DeviceName : 'Facebook Catalog' # + DeviceBrand : 'Facebook' # + OperatingSystemClass : 'Cloud' # + OperatingSystemName : 'Cloud' # + OperatingSystemVersion : '??' # + OperatingSystemVersionMajor : '??' # + OperatingSystemNameVersion : 'Cloud ??' # + OperatingSystemNameVersionMajor : 'Cloud ??' # + LayoutEngineClass : 'Robot' # + LayoutEngineName : 'facebookcatalog' # + LayoutEngineVersion : '1.0' # + LayoutEngineVersionMajor : '1' # + LayoutEngineNameVersion : 'facebookcatalog 1.0' # + LayoutEngineNameVersionMajor : 'facebookcatalog 1' # + AgentClass : 'Robot' # + AgentName : 'Facebookcatalog' # + AgentVersion : '1.0' # + AgentVersionMajor : '1' # + AgentNameVersion : 'Facebookcatalog 1.0' # + AgentNameVersionMajor : 'Facebookcatalog 1' # diff --git a/analyzer/src/main/resources/UserAgents/SpecialBrowsers.yaml b/analyzer/src/main/resources/UserAgents/SpecialBrowsers.yaml index f0fa38e585..f266308888 100644 --- a/analyzer/src/main/resources/UserAgents/SpecialBrowsers.yaml +++ b/analyzer/src/main/resources/UserAgents/SpecialBrowsers.yaml @@ -35,6 +35,7 @@ config: - matcher: require: - 'IsNull[agent.(1)product.name="Mozilla"]' + - 'IsNull[agent.(1)product.name="FakeYauaaProduct"]' extract: - 'AgentClass : 5 :"Special"' - 'AgentName : 5 :agent.(1)product.name' @@ -42,6 +43,7 @@ config: - matcher: require: - 'IsNull[agent.(1)product.name="Mozilla"]' + - 'IsNull[agent.(1)product.name="FakeYauaaProduct"]' extract: - 'AgentClass : 6 :"Special"' - 'AgentName : 6 :agent.(1)product.url' @@ -49,12 +51,14 @@ config: - matcher: require: - 'IsNull[agent.(1)product.name="Mozilla"]' + - 'IsNull[agent.(1)product.name="FakeYauaaProduct"]' extract: - 'AgentVersion : 5 :agent.(1)product.(1)comments.(1)entry.(1)text' - matcher: require: - 'IsNull[agent.(1)product.name="Mozilla"]' + - 'IsNull[agent.(1)product.name="FakeYauaaProduct"]' extract: - 'AgentVersion : 6 :agent.(1)product.version' diff --git a/analyzer/src/main/resources/UserAgents/TV.yaml b/analyzer/src/main/resources/UserAgents/TV.yaml index feb84c28d5..6f49ce4e90 100644 --- a/analyzer/src/main/resources/UserAgents/TV.yaml +++ b/analyzer/src/main/resources/UserAgents/TV.yaml @@ -1048,18 +1048,18 @@ config: OperatingSystemVersionMajor : '19' OperatingSystemNameVersion : 'Linux 19.2.39.208' OperatingSystemNameVersionMajor : 'Linux 19' - LayoutEngineClass : 'Browser' - LayoutEngineName : 'Mozilla' - LayoutEngineVersion : '5.0' - LayoutEngineVersionMajor : '5' - LayoutEngineNameVersion : 'Mozilla 5.0' - LayoutEngineNameVersionMajor : 'Mozilla 5' - AgentClass : 'Browser' - AgentName : 'Linux' - AgentVersion : 'armv7l' - AgentVersionMajor : 'armv7l' - AgentNameVersion : 'Linux armv7l' - AgentNameVersionMajor : 'Linux armv7l' + LayoutEngineClass : 'Unknown' + LayoutEngineName : 'Unknown' + LayoutEngineVersion : '??' + LayoutEngineVersionMajor : '??' + LayoutEngineNameVersion : 'Unknown ??' + LayoutEngineNameVersionMajor : 'Unknown ??' + AgentClass : 'Special' + AgentName : 'Toshiba' + AgentVersion : '??' + AgentVersionMajor : '??' + AgentNameVersion : 'Toshiba ??' + AgentNameVersionMajor : 'Toshiba ??' AgentLanguage : 'English' AgentLanguageCode : 'en' AgentSecurity : 'Strong security' diff --git a/analyzer/src/main/resources/UserAgents/Windows.yaml b/analyzer/src/main/resources/UserAgents/Windows.yaml index 5414562d61..8fdc47ae72 100644 --- a/analyzer/src/main/resources/UserAgents/Windows.yaml +++ b/analyzer/src/main/resources/UserAgents/Windows.yaml @@ -72,7 +72,7 @@ config: variable: - 'Product :agent.(1)product.(1)comments.entry.product.name="Windows NT"^' extract: - - 'DeviceClass : 50 :"Desktop"' + - 'DeviceClass : 51 :"Desktop"' - 'DeviceName : 50 :"Desktop"' - 'DeviceCpuBits : 1 :"32"' - 'OperatingSystemClass : 50 :"Desktop"' diff --git a/pom.xml b/pom.xml index b420144930..d8cae16077 100644 --- a/pom.xml +++ b/pom.xml @@ -43,7 +43,7 @@ 3.5.2 - 4.8-1 + 4.9 1.27 @@ -86,12 +86,12 @@ validate 3.1.1 - 8.37 + 8.38 4.1.3 - 4.1.4 + 4.2.0 1.7.30 2.14.0 - 5.0.0-RC9 + 5.0.2 0.13 0.8.6 5.7.0