Facebook improvements

kolonialno · Dec 5, 2020 · 6df329a · 6df329a
1 parent 26c70c4
commit 6df329a
Show file tree

Hide file tree

Showing 20 changed files with 483 additions and 112 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,7 +4,10 @@ This is intended as an overview of the major changes
 v5.21-SNAPSHOT
 ===
 - Analyzer:
+  - Improve the Calculator setup.
+- New/improved detections
   - Tests: Edge on Linux
+  - Handle Facebook agents with lists of keyvalues
 
 v5.20
 ===

diff --git a/analyzer/src/main/antlr4/nl/basjes/parse/useragent/parser/UserAgent.g4 b/analyzer/src/main/antlr4/nl/basjes/parse/useragent/parser/UserAgent.g4
@@ -384,6 +384,8 @@ productVersionWithCommas
     | base64
     | singleVersionWithCommas
     | SPECIALVERSIONWORDS
+    // This next one only occurs in a Facebook useragent
+    | CURLYBRACEOPEN keyValue ( COMMA keyValue )* CURLYBRACECLOSE
     ;
 
 

diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/AbstractUserAgentAnalyzerDirect.java b/analyzer/src/main/java/nl/basjes/parse/useragent/AbstractUserAgentAnalyzerDirect.java
@@ -71,6 +71,7 @@
 import nl.basjes.parse.useragent.analyze.treewalker.steps.walk.StepPrev;
 import nl.basjes.parse.useragent.analyze.treewalker.steps.walk.StepPrevN;
 import nl.basjes.parse.useragent.analyze.treewalker.steps.walk.StepUp;
+import nl.basjes.parse.useragent.calculate.CalculateAgentClass;
 import nl.basjes.parse.useragent.calculate.CalculateAgentEmail;
 import nl.basjes.parse.useragent.calculate.CalculateAgentName;
 import nl.basjes.parse.useragent.calculate.CalculateDeviceBrand;
@@ -115,7 +116,6 @@
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static nl.basjes.parse.useragent.UserAgent.AGENT_CLASS;
-import static nl.basjes.parse.useragent.UserAgent.AGENT_INFORMATION_EMAIL;
 import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME;
 import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME_VERSION;
 import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME_VERSION_MAJOR;
@@ -134,7 +134,6 @@
 import static nl.basjes.parse.useragent.UserAgent.LAYOUT_ENGINE_VERSION;
 import static nl.basjes.parse.useragent.UserAgent.LAYOUT_ENGINE_VERSION_MAJOR;
 import static nl.basjes.parse.useragent.UserAgent.MutableUserAgent.isSystemField;
-import static nl.basjes.parse.useragent.UserAgent.NETWORK_TYPE;
 import static nl.basjes.parse.useragent.UserAgent.OPERATING_SYSTEM_CLASS;
 import static nl.basjes.parse.useragent.UserAgent.OPERATING_SYSTEM_NAME;
 import static nl.basjes.parse.useragent.UserAgent.OPERATING_SYSTEM_NAME_VERSION;
@@ -283,6 +282,7 @@ public static void configureKryo(Object kryoInstance) {
 
         kryo.register(CalculateAgentEmail.class);
         kryo.register(CalculateAgentName.class);
+        kryo.register(CalculateAgentClass.class);
         kryo.register(CalculateDeviceBrand.class);
         kryo.register(CalculateDeviceName.class);
         kryo.register(CalculateNetworkType.class);
@@ -1627,28 +1627,35 @@ public B immediateInitialization() {
             return (B)this;
         }
 
-        private void addCalculator(FieldCalculator calculator) {
-            fieldCalculators.add(calculator);
-            if (uaa.wantedFieldNames != null) {
-                Collections.addAll(uaa.wantedFieldNames, calculator.getDependencies());
-            }
-        }
+        protected Set<String> allFieldsForWhichACalculatorExists = new HashSet<>();
 
-        private void addCalculatedMajorVersionField(String result, String dependency) {
-            if (uaa.isWantedField(result)) {
-                fieldCalculators.add(new MajorVersionCalculator(result, dependency));
+        private void registerFieldCalculator(FieldCalculator fieldCalculator) {
+            String calculatedFieldName = fieldCalculator.getCalculatedFieldName();
+            allFieldsForWhichACalculatorExists.add(calculatedFieldName);
+            if (uaa.isWantedField(calculatedFieldName)) {
+                fieldCalculators.add(fieldCalculator);
                 if (uaa.wantedFieldNames != null) {
-                    Collections.addAll(uaa.wantedFieldNames, dependency);
+                    uaa.wantedFieldNames.addAll(fieldCalculator.getDependencies());
                 }
             }
         }
 
-        private void addCalculatedConcatNONDuplicated(String result, String first, String second) {
-            if (uaa.isWantedField(result)) {
-                fieldCalculators.add(new ConcatNONDuplicatedCalculator(result, first, second));
-                if (uaa.wantedFieldNames != null) {
-                    Collections.addAll(uaa.wantedFieldNames, first, second);
+        protected void verifyCalculatorDependencyOrdering() {
+            // Verify calculator dependencies ordering
+            Set<String> seenCalculatedFields = new HashSet<>();
+            for (FieldCalculator fieldCalculator: fieldCalculators) {
+                for (String dependency: fieldCalculator.getDependencies()){
+                    if (allFieldsForWhichACalculatorExists.contains(dependency)) {
+                        if (!seenCalculatedFields.contains(dependency)) {
+                            throw new InvalidParserConfigurationException(
+                                "Calculator ordering is wrong:" +
+                                    "For " + fieldCalculator.getCalculatedFieldName() +
+                                    " we need " + dependency + " which is a " +
+                                    "calculated field but it has not yet been calculated.");
+                        }
+                    }
                 }
+                seenCalculatedFields.add(fieldCalculator.getCalculatedFieldName());
             }
         }
 
@@ -1669,40 +1676,27 @@ public UAA build() {
                 uaa.wantedFieldNames.add(DEVICE_CLASS);
             }
 
-            addCalculatedConcatNONDuplicated(AGENT_NAME_VERSION_MAJOR,              AGENT_NAME,             AGENT_VERSION_MAJOR);
-            addCalculatedConcatNONDuplicated(AGENT_NAME_VERSION,                    AGENT_NAME,             AGENT_VERSION);
-            addCalculatedMajorVersionField(AGENT_VERSION_MAJOR,                     AGENT_VERSION);
-
-            addCalculatedConcatNONDuplicated(WEBVIEW_APP_NAME_VERSION_MAJOR,        WEBVIEW_APP_NAME,       WEBVIEW_APP_VERSION_MAJOR);
-            addCalculatedMajorVersionField(WEBVIEW_APP_VERSION_MAJOR,               WEBVIEW_APP_VERSION);
+            registerFieldCalculator(new ConcatNONDuplicatedCalculator(AGENT_NAME_VERSION_MAJOR,         AGENT_NAME,             AGENT_VERSION_MAJOR));
+            registerFieldCalculator(new ConcatNONDuplicatedCalculator(AGENT_NAME_VERSION,               AGENT_NAME,             AGENT_VERSION));
+            registerFieldCalculator(new MajorVersionCalculator(AGENT_VERSION_MAJOR,                     AGENT_VERSION));
 
-            addCalculatedConcatNONDuplicated(LAYOUT_ENGINE_NAME_VERSION_MAJOR,      LAYOUT_ENGINE_NAME,     LAYOUT_ENGINE_VERSION_MAJOR);
-            addCalculatedConcatNONDuplicated(LAYOUT_ENGINE_NAME_VERSION,            LAYOUT_ENGINE_NAME,     LAYOUT_ENGINE_VERSION);
-            addCalculatedMajorVersionField(LAYOUT_ENGINE_VERSION_MAJOR,             LAYOUT_ENGINE_VERSION);
+            registerFieldCalculator(new ConcatNONDuplicatedCalculator(WEBVIEW_APP_NAME_VERSION_MAJOR,   WEBVIEW_APP_NAME,       WEBVIEW_APP_VERSION_MAJOR));
+            registerFieldCalculator(new MajorVersionCalculator(WEBVIEW_APP_VERSION_MAJOR,               WEBVIEW_APP_VERSION));
 
-            addCalculatedMajorVersionField(OPERATING_SYSTEM_NAME_VERSION_MAJOR,     OPERATING_SYSTEM_NAME_VERSION);
-            addCalculatedConcatNONDuplicated(OPERATING_SYSTEM_NAME_VERSION,         OPERATING_SYSTEM_NAME,  OPERATING_SYSTEM_VERSION);
-            addCalculatedMajorVersionField(OPERATING_SYSTEM_VERSION_MAJOR,          OPERATING_SYSTEM_VERSION);
+            registerFieldCalculator(new ConcatNONDuplicatedCalculator(LAYOUT_ENGINE_NAME_VERSION_MAJOR, LAYOUT_ENGINE_NAME,     LAYOUT_ENGINE_VERSION_MAJOR));
+            registerFieldCalculator(new ConcatNONDuplicatedCalculator(LAYOUT_ENGINE_NAME_VERSION,       LAYOUT_ENGINE_NAME,     LAYOUT_ENGINE_VERSION));
+            registerFieldCalculator(new MajorVersionCalculator(LAYOUT_ENGINE_VERSION_MAJOR,             LAYOUT_ENGINE_VERSION));
 
-            if (uaa.isWantedField(AGENT_NAME)) {
-                addCalculator(new CalculateAgentName());
-            }
-
-            if (uaa.isWantedField(NETWORK_TYPE)) {
-                addCalculator(new CalculateNetworkType());
-            }
-
-            if (uaa.isWantedField(DEVICE_NAME)) {
-                addCalculator(new CalculateDeviceName());
-            }
-
-            if (uaa.isWantedField(DEVICE_BRAND)) {
-                addCalculator(new CalculateDeviceBrand());
-            }
+            registerFieldCalculator(new MajorVersionCalculator(OPERATING_SYSTEM_NAME_VERSION_MAJOR,     OPERATING_SYSTEM_NAME_VERSION));
+            registerFieldCalculator(new ConcatNONDuplicatedCalculator(OPERATING_SYSTEM_NAME_VERSION,    OPERATING_SYSTEM_NAME,  OPERATING_SYSTEM_VERSION));
+            registerFieldCalculator(new MajorVersionCalculator(OPERATING_SYSTEM_VERSION_MAJOR,          OPERATING_SYSTEM_VERSION));
 
-            if (uaa.isWantedField(AGENT_INFORMATION_EMAIL)) {
-                addCalculator(new CalculateAgentEmail());
-            }
+            registerFieldCalculator(new CalculateAgentClass());
+            registerFieldCalculator(new CalculateAgentName());
+            registerFieldCalculator(new CalculateNetworkType());
+            registerFieldCalculator(new CalculateDeviceName());
+            registerFieldCalculator(new CalculateDeviceBrand());
+            registerFieldCalculator(new CalculateAgentEmail());
 
             Collections.reverse(fieldCalculators);
             uaa.setFieldCalculators(fieldCalculators);

diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentClass.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentClass.java
@@ -0,0 +1,54 @@
+/*
+ * Yet Another UserAgent Analyzer
+ * Copyright (C) 2013-2020 Niels Basjes
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nl.basjes.parse.useragent.calculate;
+
+import nl.basjes.parse.useragent.AgentField;
+import nl.basjes.parse.useragent.UserAgent.MutableUserAgent;
+
+import java.util.Collections;
+import java.util.Set;
+
+import static nl.basjes.parse.useragent.UserAgent.AGENT_CLASS;
+import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME;
+
+public class CalculateAgentClass extends FieldCalculator {
+    @Override
+    public void calculate(MutableUserAgent userAgent) {
+        // Cleanup the class of the useragent
+        AgentField agentClass = userAgent.get(AGENT_CLASS);
+        if (agentClass.isDefaultValue()) {
+            AgentField agentName = userAgent.get(AGENT_NAME);
+            if (!agentName.isDefaultValue()) {
+                userAgent.setForced(
+                    AGENT_CLASS,
+                    "Special",
+                    1);
+            }
+        }
+    }
+
+    @Override
+    public String getCalculatedFieldName() {
+        return AGENT_CLASS;
+    }
+
+    @Override
+    public Set<String> getDependencies() {
+        return Collections.singleton(AGENT_NAME);
+    }
+}
diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentEmail.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentEmail.java
@@ -23,7 +23,7 @@
 
 import static nl.basjes.parse.useragent.UserAgent.AGENT_INFORMATION_EMAIL;
 
-public class CalculateAgentEmail implements FieldCalculator {
+public class CalculateAgentEmail extends FieldCalculator {
     @Override
     public void calculate(MutableUserAgent userAgent) {
         // The email address is a mess
@@ -37,8 +37,7 @@ public void calculate(MutableUserAgent userAgent) {
     }
 
     @Override
-    public String toString() {
-        return "Calculate " + AGENT_INFORMATION_EMAIL;
+    public String getCalculatedFieldName() {
+        return AGENT_INFORMATION_EMAIL;
     }
-
 }
diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentName.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateAgentName.java
@@ -21,12 +21,15 @@
 import nl.basjes.parse.useragent.UserAgent.MutableUserAgent;
 import nl.basjes.parse.useragent.utils.Normalize;
 
+import java.util.Collections;
+import java.util.Set;
+
 import static nl.basjes.parse.useragent.UserAgent.AGENT_NAME;
 import static nl.basjes.parse.useragent.UserAgent.DEVICE_BRAND;
 import static nl.basjes.parse.useragent.UserAgent.NULL_VALUE;
 import static nl.basjes.parse.useragent.utils.Normalize.isLowerCase;
 
-public class CalculateAgentName implements FieldCalculator {
+public class CalculateAgentName extends FieldCalculator {
     @Override
     public void calculate(MutableUserAgent userAgent) {
         // Cleanup the name of the useragent
@@ -61,13 +64,12 @@ public void calculate(MutableUserAgent userAgent) {
     }
 
     @Override
-    public String[] getDependencies() {
-        return new String[]{DEVICE_BRAND};
+    public String getCalculatedFieldName() {
+        return AGENT_NAME;
     }
 
     @Override
-    public String toString() {
-        return "Calculate " + AGENT_NAME;
+    public Set<String> getDependencies() {
+        return Collections.singleton(DEVICE_BRAND);
     }
-
 }
diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceBrand.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceBrand.java
@@ -35,7 +35,7 @@
 import static nl.basjes.parse.useragent.utils.HostnameExtracter.extractHostname;
 import static org.apache.hc.client5.http.psl.DomainType.ICANN;
 
-public class CalculateDeviceBrand implements FieldCalculator {
+public class CalculateDeviceBrand extends FieldCalculator {
 
     private final Set<String> unwantedUrlBrands;
     private final Set<String> unwantedEmailBrands;
@@ -168,13 +168,12 @@ private String extractCompanyFromHostName(String hostname, Set<String> blackList
     }
 
     @Override
-    public String[] getDependencies() {
-        return new String[]{AGENT_INFORMATION_URL, AGENT_INFORMATION_EMAIL};
+    public String getCalculatedFieldName() {
+        return DEVICE_BRAND;
     }
 
     @Override
-    public String toString() {
-        return "Calculate " + DEVICE_BRAND;
+    public Set<String> getDependencies() {
+        return new HashSet<>(Arrays.asList(AGENT_INFORMATION_URL, AGENT_INFORMATION_EMAIL));
     }
-
 }
diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceName.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateDeviceName.java
@@ -21,6 +21,8 @@
 import nl.basjes.parse.useragent.UserAgent.MutableUserAgent;
 import nl.basjes.parse.useragent.utils.Normalize;
 
+import java.util.Collections;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 import static java.util.regex.Pattern.CASE_INSENSITIVE;
@@ -29,7 +31,7 @@
 import static nl.basjes.parse.useragent.UserAgent.DEVICE_NAME;
 import static nl.basjes.parse.useragent.UserAgent.UNKNOWN_VALUE;
 
-public class CalculateDeviceName implements FieldCalculator {
+public class CalculateDeviceName extends FieldCalculator {
 
 
     private static final Pattern CLEAN_1_PATTERN = Pattern.compile("AppleWebKit", CASE_INSENSITIVE | LITERAL);
@@ -64,8 +66,13 @@ public void calculate(MutableUserAgent userAgent) {
     }
 
     @Override
-    public String[] getDependencies() {
-        return new String[]{DEVICE_BRAND};
+    public String getCalculatedFieldName() {
+        return DEVICE_NAME;
+    }
+
+    @Override
+    public Set<String> getDependencies() {
+        return Collections.singleton(DEVICE_BRAND);
     }
 
     @Override

diff --git a/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateNetworkType.java b/analyzer/src/main/java/nl/basjes/parse/useragent/calculate/CalculateNetworkType.java
@@ -23,7 +23,7 @@
 
 import static nl.basjes.parse.useragent.UserAgent.NETWORK_TYPE;
 
-public class CalculateNetworkType implements FieldCalculator {
+public class CalculateNetworkType extends FieldCalculator {
 
     @Override
     public void calculate(MutableUserAgent userAgent) {
@@ -37,6 +37,11 @@ public void calculate(MutableUserAgent userAgent) {
         }
     }
 
+    @Override
+    public String getCalculatedFieldName() {
+        return NETWORK_TYPE;
+    }
+
     @Override
     public String toString() {
         return "Calculate " + NETWORK_TYPE;

diff --git a/...yzer/src/main/java/nl/basjes/parse/useragent/calculate/ConcatNONDuplicatedCalculator.java b/...yzer/src/main/java/nl/basjes/parse/useragent/calculate/ConcatNONDuplicatedCalculator.java
@@ -20,9 +20,13 @@
 import nl.basjes.parse.useragent.AgentField;
 import nl.basjes.parse.useragent.UserAgent.MutableUserAgent;
 
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
 import static nl.basjes.parse.useragent.UserAgent.NULL_VALUE;
 
-public class ConcatNONDuplicatedCalculator implements FieldCalculator {
+public class ConcatNONDuplicatedCalculator extends FieldCalculator {
 
     private final String targetName;
     private final String firstName;
@@ -78,8 +82,13 @@ public void calculate(MutableUserAgent userAgent) {
     }
 
     @Override
-    public String[] getDependencies() {
-        return new String[]{firstName, secondName};
+    public String getCalculatedFieldName() {
+        return targetName;
+    }
+
+    @Override
+    public Set<String> getDependencies() {
+        return new HashSet<>(Arrays.asList(firstName, secondName));
     }
 
     @Override