Document malware analysis by Diario (#98)

ElevenPaths · Apr 3, 2020 · 1f8f55f · 1f8f55f
1 parent a80d28c
commit 1f8f55f
Show file tree

Hide file tree

Showing 30 changed files with 5,186 additions and 4,427 deletions.
diff --git a/FOCA/Analysis/DiarioAnalyzer.cs b/FOCA/Analysis/DiarioAnalyzer.cs
@@ -0,0 +1,151 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Schedulers;
+
+namespace FOCA.Analysis
+{
+    public class DiarioAnalyzer
+    {
+        private const int MaxRetries = 10;
+        private const int FileNotFoundErrorCode = 406;
+        private const string Analyzed = "A";
+        private const string Processing = "P";
+        private const string Queued = "Q";
+        private const string Failed = "F";
+
+        private static readonly TimeSpan DelayBetweenRetries = TimeSpan.FromSeconds(2);
+        private DiarioSDKNet.Diario sdk;
+
+        private TaskScheduler currentScheduler;
+
+        public static readonly string[] SupportedExtensions = new string[] { ".docx", ".xlsx", ".doc", ".xls", ".pdf" };
+
+        public DiarioAnalyzer(string apiKey, string secret)
+        {
+            if (String.IsNullOrWhiteSpace(apiKey))
+                throw new ArgumentNullException(nameof(apiKey));
+
+            if (String.IsNullOrWhiteSpace(secret))
+                throw new ArgumentNullException(nameof(secret));
+
+            this.sdk = new DiarioSDKNet.Diario(apiKey, secret);
+            this.currentScheduler = new QueuedTaskScheduler(TaskScheduler.Default, Environment.ProcessorCount);
+        }
+
+        private static string NormalizeExtension(string extension)
+        {
+            if (String.IsNullOrWhiteSpace(extension))
+                throw new ArgumentNullException(nameof(extension));
+
+            if (!extension.StartsWith("."))
+                extension = "." + extension;
+
+            return extension.ToLowerInvariant().Trim();
+        }
+
+        public static bool IsSupportedExtension(string extension)
+        {
+            if (String.IsNullOrWhiteSpace(extension))
+                return false;
+
+            string normalizedExtension = NormalizeExtension(extension);
+            return SupportedExtensions.Any(p => p.Equals(normalizedExtension));
+        }
+
+        public void CheckMalware(string filePath, Action<DiarioFileAnalysis> finishCallback, CancellationToken token = default(CancellationToken))
+        {
+            this.CheckMalware(new DiarioFileAnalysis(filePath, finishCallback, token));
+        }
+
+        public void CheckMalware(DiarioFileAnalysis file)
+        {
+            if (file == null)
+                throw new ArgumentNullException(nameof(file));
+
+            Task.Factory.StartNew(async () => await CheckFile(file), file.CancelToken, TaskCreationOptions.DenyChildAttach | TaskCreationOptions.HideScheduler, this.currentScheduler);
+        }
+
+        private async Task CheckFile(DiarioFileAnalysis file)
+        {
+            try
+            {
+                if (file.Retries > MaxRetries || file.CancelToken.IsCancellationRequested)
+                {
+                    file.Error = file.Retries > MaxRetries ? "Too many retries. Try again later" : "Operation canceled";
+                    file.Callback(file);
+                }
+                else
+                {
+                    byte[] fileContent = null;
+                    if (String.IsNullOrWhiteSpace(file.Sha256Hash))
+                    {
+                        using (SHA256Managed sha256 = new SHA256Managed())
+                        {
+                            fileContent = File.ReadAllBytes(file.FilePath);
+                            file.Sha256Hash = BitConverter.ToString(sha256.ComputeHash(fileContent)).Replace("-", "").ToLowerInvariant();
+                        }
+                    }
+
+                    BaseSDK.ApiResponse<dynamic> diarioResponse = this.sdk.Search(file.Sha256Hash);
+                    if (diarioResponse != null)
+                    {
+                        if (diarioResponse.Error != null)
+                        {
+                            if (diarioResponse.Error?.Code == FileNotFoundErrorCode)
+                            {
+                                diarioResponse = this.sdk.Upload(fileContent ?? File.ReadAllBytes(file.FilePath), Path.GetFileName(file.FilePath));
+                                file.Retries++;
+                                await Task.Delay(DelayBetweenRetries);
+                                this.CheckMalware(file);
+                            }
+                            else
+                            {
+                                file.Error = diarioResponse.Error.Message;
+                                file.Callback(file);
+                            }
+                        }
+                        else if (diarioResponse.Data?["status"] == Analyzed)
+                        {
+                            file.Prediction = DiarioSDKNet.Diario.GetPredictonFromString(diarioResponse.Data["prediction"]);
+                            file.Completed = true;
+                            file.Callback(file);
+                        }
+                        else if (diarioResponse.Data?["status"] == Processing || diarioResponse.Data?["status"] == Queued)
+                        {
+                            file.Retries++;
+                            await Task.Delay(DelayBetweenRetries);
+                            this.CheckMalware(file);
+                        }
+                        else if (diarioResponse.Data?["status"] == Failed)
+                        {
+                            file.Error = "The analysis failed";
+                            file.Prediction = DiarioSDKNet.Diario.Prediction.Unknown;
+                            file.Callback(file);
+                        }
+                        else
+                        {
+                            file.Error = "Unknown status";
+                            file.Prediction = DiarioSDKNet.Diario.Prediction.Unknown;
+                            file.Callback(file);
+                        }
+                    }
+                    else
+                    {
+                        file.Error = "No response from DIARIO service";
+                        file.Callback(file);
+                    }
+                }
+            }
+            catch (Exception e)
+            {
+                file.Error = e.Message;
+                file.Completed = false;
+                file.Callback(file);
+            }
+        }
+    }
+}
diff --git a/FOCA/Analysis/DiarioFileAnalysis.cs b/FOCA/Analysis/DiarioFileAnalysis.cs
@@ -0,0 +1,39 @@
+using System;
+using System.Threading;
+
+namespace FOCA.Analysis
+{
+    public class DiarioFileAnalysis
+    {
+        public string FilePath { get; private set; }
+
+        public int Retries { get; set; }
+
+        public Action<DiarioFileAnalysis> Callback { get; set; }
+
+        public DiarioSDKNet.Diario.Prediction Prediction { get; set; }
+
+        public bool Completed { get; set; }
+
+        public string Error { get; set; }
+
+        public string Sha256Hash { get; set; }
+
+        public CancellationToken CancelToken { get; private set; }
+
+        public DiarioFileAnalysis(string file, Action<DiarioFileAnalysis> callback, CancellationToken token = default(CancellationToken))
+        {
+            if (String.IsNullOrWhiteSpace(file))
+                throw new ArgumentNullException(nameof(file));
+
+            if (callback == null)
+                throw new ArgumentNullException(nameof(callback));
+
+            this.FilePath = file;
+            this.Callback = callback;
+            this.Retries = 0;
+            this.CancelToken = token;
+            this.Completed = false;
+        }
+    }
+}
diff --git a/FOCA/Configuration.cs b/FOCA/Configuration.cs
@@ -1,12 +1,10 @@
+using FOCA.Analysis.FingerPrinting;
+using FOCA.Database.Controllers;
+using FOCA.Plugins;
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using System.Windows.Forms;
-using FOCA.Analysis.FingerPrinting;
-using FOCA.Database.Controllers;
-using FOCA.Plugins;
-using Newtonsoft.Json.Linq;
 
 namespace FOCA
 {
@@ -65,6 +63,10 @@ public class Configuration
         /// </summary>
         public string GoogleApiKey { get; set; }
 
+        public string DiarioAPIKey { get; set; }
+
+        public string DiarioAPISecret { get; set; }
+
         /// <summary>
         ///     Limit max recursivity level during search process
         /// </summary>

diff --git a/FOCA/Database/Controllers/ConfigurationController.cs b/FOCA/Database/Controllers/ConfigurationController.cs
@@ -32,6 +32,8 @@ public void Save(Configuration item)
                     config.SelectedTechExtensions = item.SelectedTechExtensions;
                     config.SPathsPlugins = item.SPathsPlugins;
                     config.ShodanApiKey = item.ShodanApiKey;
+                    config.DiarioAPIKey = item.DiarioAPIKey;
+                    config.DiarioAPISecret = item.DiarioAPISecret;
 
                     context.SaveChanges();
                 }
@@ -78,7 +80,9 @@ private Configuration LoadDefaultConfig()
                 SimultaneousDownloads = 15,
                 SPathsPlugins = string.Empty,
                 UseAllDns = true,
-                webSearcherEngine = 2
+                webSearcherEngine = 2,
+                DiarioAPIKey = "LfUxRse6mjsRA3DZNBuq",
+                DiarioAPISecret = "ThyDYxcaGdHHK6TC3YcMqG2kFexF4D8djmdpcMy8"
             };
 
             using (FocaContextDb context = new FocaContextDb())

diff --git a/FOCA/Database/Entities/Ficheros.cs b/FOCA/Database/Entities/Ficheros.cs
@@ -1,6 +1,6 @@
-using System;
 using FOCA.ModifiedComponents;
 using Newtonsoft.Json;
+using System;
 
 namespace FOCA.Database.Entities
 {
@@ -83,6 +83,8 @@ public class FilesItem : BaseItem
         public DateTime Date { get; set; } = DateTime.Now;
         public int Size { get; set; }
         public bool Downloaded { get; set; }
-        public bool Processed { get; set; }
+        public bool MetadataExtracted { get; set; }
+        public bool DiarioAnalyzed { get; set; }
+        public string DiarioPrediction { get; set; } = "Unanalyzed";
     }
 }
diff --git a/FOCA/Database/Migrations/202001200633256_MalwareAnalysis.Designer.cs b/FOCA/Database/Migrations/202001200633256_MalwareAnalysis.Designer.cs
diff --git a/FOCA/Database/Migrations/202001200633256_MalwareAnalysis.cs b/FOCA/Database/Migrations/202001200633256_MalwareAnalysis.cs
@@ -0,0 +1,26 @@
+namespace FOCA.Migrations
+{
+    using System.Data.Entity.Migrations;
+
+    public partial class MalwareAnalysis : DbMigration
+    {
+        public override void Up()
+        {
+            AddColumn("dbo.Configurations", "DiarioAPIKey", c => c.String());
+            AddColumn("dbo.Configurations", "DiarioAPISecret", c => c.String());
+            RenameColumn("dbo.FilesItems", "Processed", "MetadataExtracted");
+            AddColumn("dbo.FilesItems", "DiarioAnalyzed", c => c.Boolean(nullable: false));
+            AddColumn("dbo.FilesItems", "DiarioPrediction", c => c.String());
+            Sql("UPDATE dbo.Configurations SET DiarioAPIKey='LfUxRse6mjsRA3DZNBuq', DiarioAPISecret='ThyDYxcaGdHHK6TC3YcMqG2kFexF4D8djmdpcMy8'");
+        }
+
+        public override void Down()
+        {
+            RenameColumn("dbo.FilesItems", "MetadataExtracted", "Processed");
+            DropColumn("dbo.FilesItems", "DiarioPrediction");
+            DropColumn("dbo.FilesItems", "DiarioAnalyzed");
+            DropColumn("dbo.Configurations", "DiarioAPISecret");
+            DropColumn("dbo.Configurations", "DiarioAPIKey");
+        }
+    }
+}