mutate: coverage import

plugin/mutate/source/dextool/plugin/mutate/backend/database/schema.d

@@ -114,7 +114,6 @@ immutable schemaMutantQTable = "schema_mutant_q";
 immutable schemaMutantV2Table = "schemata_mutant_v2";
 immutable schemaSizeQTable = "schema_size_q";
 immutable schemaVersionTable = "schema_version";
-immutable srcCovInfoTable = "src_cov_info";
 immutable srcCovTable = "src_cov_instr";
 immutable srcCovTimeStampTable = "src_cov_timestamp";
 immutable srcMetadataTable = "src_metadata";
@@ -124,6 +123,7 @@ immutable testCmdRelMutantTable = "test_cmd_rel_mutant";
 immutable testCmdTable = "test_cmd";
 immutable testFilesTable = "test_files";
 
+private immutable srcCovInfoTable = "src_cov_info";
 private immutable schemataTable = "schemata";
 private immutable schemataMutantTable = "schemata_mutant";
 private immutable schemataUsedTable = "schemata_used";
@@ -654,24 +654,14 @@ struct CoverageCodeRegionTable {
     @ColumnName("file_id")
     long fileId;
 
+    /// True if the region has been visited.
+    bool status;
+
     /// the region in the files, in bytes.
     uint begin;
     uint end;
 }
 
-/** Each coverage region that has a valid status has an entry in this table.
- * It do mean that there can be region that do not exist in this table. That
- * mean that something went wrong when gathering the data.
- */
-@TableName(srcCovInfoTable)
-@TableForeignKey("id", KeyRef("src_cov_instr(id)"), KeyParam("ON DELETE CASCADE"))
-struct CoverageInfoTable {
-    long id;
-
-    /// True if the region has been visited.
-    bool status;
-}
-
 /// When the coverage information was gathered.
 @TableName(srcCovTimeStampTable)
 struct CoverageTimeTtampTable {
@@ -876,9 +866,8 @@ void upgradeV0(ref Miniorm db) {
             RuntimeHistoryTable,
             MutationScoreHistoryTable,
             MutationFileScoreHistoryTable, TestFilesTable, CoverageCodeRegionTable,
-            CoverageInfoTable, CoverageTimeTtampTable, DependencyFileTable,
+            CoverageTimeTtampTable, DependencyFileTable,
             DependencyRootTable, DextoolVersionTable, ConfigVersionTable,
-            TestCmdOriginalTable,
             TestCmdMutatedTable,
             MutantMemOverloadtWorklistTbl, TestCmdRelMutantTable,
             TestCmdTable, SchemaMutantV2Table, SchemaFragmentV2Table));
@@ -1750,6 +1739,20 @@ void upgradeV30(ref Miniorm db) {
 
 /// 2020-12-29
 void upgradeV31(ref Miniorm db) {
+    @TableName(srcCovTable)
+    @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE"))
+    @TableConstraint("file_offset UNIQUE (file_id, begin, end)")
+    struct CoverageCodeRegionTable {
+        long id;
+
+        @ColumnName("file_id")
+        long fileId;
+
+        /// the region in the files, in bytes.
+        uint begin;
+        uint end;
+    }
+
     @TableName(srcCovInfoTable)
     @TableForeignKey("id", KeyRef("src_cov_info(id)"), KeyParam("ON DELETE CASCADE"))
     struct CoverageInfoTable {
@@ -2073,6 +2076,19 @@ void upgradeV50(ref Miniorm db) {
 
 // 2022-06-14
 void upgradeV51(ref Miniorm db) {
+    /** Each coverage region that has a valid status has an entry in this table.
+     * It do mean that there can be region that do not exist in this table. That
+     * mean that something went wrong when gathering the data.
+     */
+    @TableName(srcCovInfoTable)
+    @TableForeignKey("id", KeyRef("src_cov_instr(id)"), KeyParam("ON DELETE CASCADE"))
+    struct CoverageInfoTable {
+        long id;
+
+        /// True if the region has been visited.
+        bool status;
+    }
+
     db.run("DROP TABLE " ~ srcCovInfoTable);
     db.run(buildSchema!CoverageInfoTable);
 }

plugin/mutate/source/dextool/plugin/mutate/backend/database/standalone.d

@@ -2060,6 +2060,22 @@ struct DbCoverage {
         }
     }
 
+    /// Add coverage regions with status.
+    void putCoverageMap(const Path file, const Offset[] region, const bool[] statuses) @trusted {
+        static immutable sql = "INSERT OR IGNORE INTO " ~ srcCovTable
+            ~ " (file_id, status, begin, end)
+            VALUES((SELECT id FROM " ~ filesTable ~ " WHERE path=:path), :status, :begin, :end)";
+        auto stmt = db.prepare(sql);
+        stmt.get.bind(":path", file);
+        foreach (i, a; region) {
+            stmt.get.bind(":status", statuses[i]);
+            stmt.get.bind(":begin", a.begin);
+            stmt.get.bind(":end", a.end);
+            stmt.get.execute;
+            stmt.get.reset;
+        }
+    }
+
     CovRegion[][FileId] getCoverageMap() @trusted {
         static immutable sql = "SELECT file_id, begin, end, id FROM " ~ srcCovTable;
         auto stmt = db.prepare(sql);
@@ -2079,9 +2095,7 @@ struct DbCoverage {
     }
 
     CovRegionStatus[] getCoverageStatus(FileId fileId) @trusted {
-        immutable sql = "SELECT t0.begin, t0.end, t1.status FROM "
-            ~ srcCovTable ~ " t0, " ~ srcCovInfoTable ~ " t1
-        WHERE t0.id = t1.id AND t0.file_id = :fid";
+        immutable sql = "SELECT begin, end, status FROM " ~ srcCovTable ~ " WHERE file_id = :fid";
         auto stmt = db.prepare(sql);
         stmt.get.bind(":fid", fileId.get);
         auto rval = appender!(CovRegionStatus[])();
@@ -2107,9 +2121,8 @@ struct DbCoverage {
         stmt.get.execute;
     }
 
-    void putCoverageInfo(const CoverageRegionId regionId, bool status) {
-        static immutable sql = "INSERT OR REPLACE INTO " ~ srcCovInfoTable
-            ~ " (id, status) VALUES(:id, :status)";
+    void putCoverageStatus(const CoverageRegionId regionId, const bool status) @trusted {
+        static immutable sql = "UPDATE " ~ srcCovTable ~ " SET status = :status WHERE id = :id";
         auto stmt = db.prepare(sql);
         stmt.get.bind(":id", regionId.get);
         stmt.get.bind(":status", status);
@@ -2136,13 +2149,12 @@ struct DbCoverage {
     }
 
     MutationStatusId[] getNotCoveredMutants() @trusted {
-        static immutable sql = format!"SELECT DISTINCT t3.st_id FROM %1$s t0, %2$s t1, %3$s t2, %4$s t3
+        static immutable sql = format!"SELECT DISTINCT t2.st_id FROM %1$s t0, %2$s t1, %3$s t2
             WHERE t0.status = 0 AND
-            t0.id = t1.id AND
-            t1.file_id = t2.file_id AND
-            (t2.offset_begin BETWEEN t1.begin AND t1.end) AND
-            (t2.offset_end BETWEEN t1.begin AND t1.end) AND
-            t2.id = t3.mp_id"(srcCovInfoTable, srcCovTable, mutationPointTable, mutationTable);
+            t0.file_id = t1.file_id AND
+            (t1.offset_begin BETWEEN t0.begin AND t0.end) AND
+            (t1.offset_end BETWEEN t0.begin AND t0.end) AND
+            t1.id = t2.mp_id"(srcCovTable, mutationPointTable, mutationTable);
 
         auto app = appender!(MutationStatusId[])();
         auto stmt = db.prepare(sql);

plugin/mutate/source/dextool/plugin/mutate/backend/test_mutant/coverage.d

@@ -7,7 +7,7 @@ module dextool.plugin.mutate.backend.test_mutant.coverage;
 
 import core.time : Duration;
 import logger = std.experimental.logger;
-import std.algorithm : map, filter, sort;
+import std.algorithm : map, filter, sort, canFind;
 import std.array : array, appender, empty;
 import std.exception : collectException;
 import std.stdio : File;
@@ -27,7 +27,7 @@ import dextool.plugin.mutate.backend.database : CovRegion, CoverageRegionId, Fil
 import dextool.plugin.mutate.backend.database : Database;
 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob;
 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner : TestRunner, TestResult;
-import dextool.plugin.mutate.backend.type : Mutation, Language;
+import dextool.plugin.mutate.backend.type : Mutation, Language, Offset;
 import dextool.plugin.mutate.type : ShellCommand, UserRuntime, CoverageRuntime;
 import dextool.plugin.mutate.config : ConfigCoverage;
 
@@ -44,6 +44,10 @@ struct CoverageDriver {
         bool hasRoot;
     }
 
+    static struct ImportJSON {
+
+    }
+
     static struct SaveOriginal {
     }
 
@@ -71,8 +75,8 @@ struct CoverageDriver {
     static struct Done {
     }
 
-    alias Fsm = my.fsm.Fsm!(None, Initialize, InitializeRoots, SaveOriginal,
-            Instrument, Compile, Run, SaveToDb, Restore, Done);
+    alias Fsm = my.fsm.Fsm!(None, ImportJSON, Initialize, InitializeRoots,
+            SaveOriginal, Instrument, Compile, Run, SaveToDb, Restore, Done);
 
     private {
         Fsm fsm;
@@ -108,6 +112,10 @@ struct CoverageDriver {
         Set!AbsolutePath roots;
 
         CoverageRuntime runtime;
+
+        bool importExists = false;
+
+        ConfigCoverage.CoverageMetaData metaData;
     }
 
     this(FilesysIO fio, Database* db, TestRunner* runner, ConfigCoverage conf,
@@ -119,6 +127,7 @@ struct CoverageDriver {
         this.buildCmdTimeout = buildCmdTimeout;
         this.log = conf.log;
         this.runtime = conf.runtime;
+        this.metaData = conf.metaData.orElse(ConfigCoverage.CoverageMetaData.init);
 
         foreach (a; conf.userRuntimeCtrl) {
             auto p = fio.toAbsoluteRoot(a.file);
@@ -131,7 +140,11 @@ struct CoverageDriver {
     }
 
     static void execute_(ref CoverageDriver self) @trusted {
-        self.fsm.next!((None a) => Initialize.init, (Initialize a) {
+        self.fsm.next!((None a) => ImportJSON.init, (ImportJSON a) {
+            if (self.importExists)
+                return fsm(Done.init);
+            return fsm(Initialize.init);
+        }, (Initialize a) {
             if (self.runtime == CoverageRuntime.inject)
                 return fsm(InitializeRoots.init);
             return fsm(SaveOriginal.init);
@@ -238,6 +251,117 @@ nothrow:
         }
     }
 
+    void opCall(ImportJSON data) {
+        import std.file : exists, readText;
+        import std.json : JSONValue, JSONOptions, parseJSON;
+        import std.string : startsWith;
+
+        if (metaData.get.empty)
+            return;
+
+        auto metadataPath = metaData.get;
+
+        if (!exists(metadataPath)) {
+            logger.error("File: " ~ metadataPath ~ " does not exist").collectException;
+            return;
+        }
+
+        string fileContent;
+        try {
+            fileContent = readText(metadataPath);
+        } catch (Exception e) {
+            logger.error("Unable to read file " ~ metadataPath).collectException;
+            return;
+        }
+
+        JSONValue jContent;
+        try {
+            jContent = parseJSON(fileContent, JSONOptions.doNotEscapeSlashes);
+        } catch (Exception e) {
+            logger.info(e.msg).collectException;
+            logger.error("Failed to parse filecontect of " ~ metadataPath ~ " into JSON")
+                .collectException;
+            return;
+        }
+
+        JSONValue objectData;
+        try {
+            objectData = jContent["coverage-info"];
+        } catch (Exception e) {
+            logger.info(e.msg).collectException;
+            logger.error("Object 'coverage-info' not found in file " ~ metadataPath)
+                .collectException;
+            return;
+        }
+
+        JSONValue filesObjectData;
+        try {
+            filesObjectData = objectData["file-keys"];
+        } catch (Exception e) {
+            logger.info(e.msg).collectException;
+            logger.error("Object 'file-keys' not found in file " ~ metadataPath).collectException;
+            return;
+        }
+        importExists = true;
+
+        try {
+            foreach (filePath; filesObjectData.arrayNoRef) {
+                auto fileData = objectData[filePath.str];
+
+                auto p = fio.toAbsoluteRoot(Path(filePath.str));
+                auto r = fio.getOutputDir();
+
+                if (p.toString.startsWith(r.toString)) {
+                    auto cov = fileData["coverage"];
+                    long[] covLines;
+                    foreach (line; cov.arrayNoRef) {
+                        covLines ~= line.integer;
+                    }
+                    auto noCov = fileData["no_coverage"];
+                    long[] noCovLines;
+                    foreach (line; noCov.arrayNoRef) {
+                        noCovLines ~= line.integer;
+                    }
+
+                    auto blob = fio.makeInput(p);
+
+                    Offset[] regions;
+                    bool[] statuses;
+                    int byteCounter = 0;
+                    int lineCounter = 1;
+                    uint lineStart = 0;
+                    uint lineEnd;
+                    foreach (b; blob.content) {
+                        if (b == '\n') {
+                            lineEnd = byteCounter;
+                            if (covLines.canFind(lineCounter) || noCovLines.canFind(lineCounter)) {
+                                Offset region;
+                                region.begin = lineStart;
+                                region.end = lineEnd;
+                                regions ~= region;
+                                if (covLines.canFind(lineCounter)) {
+                                    statuses ~= true;
+                                } else {
+                                    statuses ~= false;
+                                }
+                            }
+                            lineStart = byteCounter + 1;
+                            lineCounter++;
+                        }
+                        byteCounter++;
+                    }
+                    spinSql!(() @trusted {
+                        db.coverageApi.putCoverageMap(Path(filePath.str), regions, statuses);
+                    });
+                }
+            }
+        } catch (Exception e) {
+            logger.warning(e.msg).collectException;
+            logger.error("Faulty iteration").collectException;
+            return;
+        }
+    }
+
     void opCall(SaveOriginal data) {
         try {
             restore = regions.byKey.array;
@@ -352,7 +476,7 @@ nothrow:
         void save() @trusted {
             auto trans = db.transaction;
             foreach (a; data.covMap) {
-                db.coverageApi.putCoverageInfo(localId[a.id], a.status);
+                db.coverageApi.putCoverageStatus(localId[a.id], a.status);
             }
             db.coverageApi.updateCoverageTimeStamp;
             trans.commit;

plugin/mutate/source/dextool/plugin/mutate/backend/test_mutant/package.d

@@ -1563,7 +1563,7 @@ nothrow:
         auto covTimeStamp = spinSql!(() => db.coverageApi.getCoverageTimeStamp).orElse(
                 SysTime.init);
 
-        if (tracked < covTimeStamp) {
+        if (tracked < covTimeStamp && !covConf.metaData.hasValue) {
             logger.info("Coverage information is up to date").collectException;
             return;
         } else {

plugin/mutate/source/dextool/plugin/mutate/config.d

@@ -332,6 +332,10 @@ struct ConfigCoverage {
     /// If the generated coverage files should be saved.
     bool log;
 
+    alias CoverageMetaData = NamedType!(AbsolutePath, Tag!"CoverageMetaData",
+            AbsolutePath.init, TagStringable);
+    Optional!CoverageMetaData metaData;
+
     /// allows a user to control exactly which files the coverage and schemata
     /// runtime is injected in.
     UserRuntime[] userRuntimeCtrl;