support for other coverage data with metadata option for mutate test.…

… Coverage in DB changed so that status is stored in instr

plugin/mutate/source/dextool/plugin/mutate/backend/database/schema.d

@@ -115,7 +115,6 @@ immutable schemataFragmentTable = "schemata_fragment";
 immutable schemataMutantTable = "schemata_mutant";
 immutable schemataTable = "schemata";
 immutable schemataUsedTable = "schemata_used";
-immutable srcCovInfoTable = "src_cov_info";
 immutable srcCovTable = "src_cov_instr";
 immutable srcCovTimeStampTable = "src_cov_timestamp";
 immutable srcMetadataTable = "src_metadata";
@@ -125,6 +124,7 @@ immutable testCmdRelMutantTable = "test_cmd_rel_mutant";
 immutable testCmdTable = "test_cmd";
 immutable testFilesTable = "test_files";
 
+private immutable srcCovInfoTable = "src_cov_info";
 private immutable invalidSchemataTable = "invalid_schemata";
 private immutable schemataWorkListTable = "schemata_worklist";
 private immutable testCaseTableV1 = "test_case";
@@ -677,6 +677,9 @@ struct CoverageCodeRegionTable {
     @ColumnName("file_id")
     long fileId;
 
+    /// True if the region has been visited.
+    bool status;
+
     /// the region in the files, in bytes.
     uint begin;
     uint end;
@@ -1722,6 +1725,20 @@ void upgradeV30(ref Miniorm db) {
 
 /// 2020-12-29
 void upgradeV31(ref Miniorm db) {
+    @TableName(srcCovTable)
+    @TableForeignKey("file_id", KeyRef("files(id)"), KeyParam("ON DELETE CASCADE"))
+    @TableConstraint("file_offset UNIQUE (file_id, begin, end)")
+    struct CoverageCodeRegionTable {
+        long id;
+
+        @ColumnName("file_id")
+        long fileId;
+
+        /// the region in the files, in bytes.
+        uint begin;
+        uint end;
+    }
+
     @TableName(srcCovInfoTable)
     @TableForeignKey("id", KeyRef("src_cov_info(id)"), KeyParam("ON DELETE CASCADE"))
     struct CoverageInfoTable {
@@ -2026,8 +2043,7 @@ void upgradeV52(ref Miniorm db) {
 
 // 2022-08-10
 void upgradeV53(ref Miniorm db) {
-    @TableName(mutationStatusTable)
-    @TableConstraint("checksum UNIQUE (checksum)")
+    @TableName(mutationStatusTable) @TableConstraint("checksum UNIQUE (checksum)")
     struct MutationStatusTbl {
         long id;
 
@@ -2117,6 +2133,12 @@ void upgradeV55(ref Miniorm db) {
     db.run(buildSchema!MutationStatusTbl);
 }
 
+void upgradeV56(ref Miniorm db) {
+    db.run("DROP TABLE " ~ srcCovInfoTable);
+    db.run("DROP TABLE " ~ srcCovTable);
+    db.run(buildSchema!CoverageCodeRegionTable);
+}
+
 void replaceTbl(ref Miniorm db, string src, string dst) {
     db.run("DROP TABLE " ~ dst);
     db.run("ALTER TABLE " ~ src ~ " RENAME TO " ~ dst);

plugin/mutate/source/dextool/plugin/mutate/backend/database/standalone.d

@@ -2334,6 +2334,24 @@ struct DbCoverage {
         }
     }
 
+    /// Add coverage regions with status.
+    void putCoverageMap(const Path filePath, const Offset[] region, const bool[] statuses) @trusted {
+        static immutable sql = format!"INSERT OR IGNORE INTO %1$s (file_id, status, begin, end)
+            VALUES((SELECT id FROM %2$s WHERE path = :path), :status, :begin, :end)"(srcCovTable,
+                filesTable);
+        auto stmt = db.prepare(sql);
+        int i = 0;
+        stmt.get.bind(":path", filePath);
+        foreach (a; region) {
+            stmt.get.bind(":status", statuses[i]);
+            stmt.get.bind(":begin", a.begin);
+            stmt.get.bind(":end", a.end);
+            stmt.get.execute;
+            stmt.get.reset;
+            i++;
+        }
+    }
+
     CovRegion[][FileId] getCoverageMap() @trusted {
         static immutable sql = format!"SELECT file_id, begin, end, id FROM %s"(srcCovTable);
         auto stmt = db.prepare(sql);
@@ -2352,10 +2370,16 @@ struct DbCoverage {
         return rval;
     }
 
+    void putCoverageStatus(const CoverageRegionId regionId, bool status) {
+        static immutable sql = format!"UPDATE %1s SET status = :status WHERE id = :id"(srcCovTable);
+        auto stmt = db.prepare(sql);
+        stmt.get.bind(":id", regionId.get);
+        stmt.get.bind(":status", status);
+        stmt.get.execute;
+    }
+
     CovRegionStatus[] getCoverageStatus(FileId fileId) @trusted {
-        immutable sql = "SELECT t0.begin, t0.end, t1.status FROM "
-            ~ srcCovTable ~ " t0, " ~ srcCovInfoTable ~ " t1
-        WHERE t0.id = t1.id AND t0.file_id = :fid";
+        immutable sql = "SELECT begin, end, status FROM " ~ srcCovTable ~ " WHERE file_id = :fid";
         auto stmt = db.prepare(sql);
         stmt.get.bind(":fid", fileId.get);
         auto rval = appender!(CovRegionStatus[])();
@@ -2381,15 +2405,6 @@ struct DbCoverage {
         stmt.get.execute;
     }
 
-    void putCoverageInfo(const CoverageRegionId regionId, bool status) {
-        static immutable sql = format!"INSERT OR REPLACE INTO %1$s (id, status) VALUES(:id, :status)"(
-                srcCovInfoTable);
-        auto stmt = db.prepare(sql);
-        stmt.get.bind(":id", regionId.get);
-        stmt.get.bind(":status", status);
-        stmt.get.execute;
-    }
-
     Optional!SysTime getCoverageTimeStamp() @trusted {
         static immutable sql = format!"SELECT timeStamp FROM %s"(srcCovTimeStampTable);
         auto stmt = db.prepare(sql);
@@ -2410,13 +2425,12 @@ struct DbCoverage {
     }
 
     MutationStatusId[] getNotCoveredMutants() @trusted {
-        static immutable sql = format!"SELECT DISTINCT t3.st_id FROM %1$s t0, %2$s t1, %3$s t2, %4$s t3
+        static immutable sql = format!"SELECT DISTINCT t2.st_id FROM %1$s t0, %2$s t1, %3$s t2
             WHERE t0.status = 0 AND
-            t0.id = t1.id AND
-            t1.file_id = t2.file_id AND
-            (t2.offset_begin BETWEEN t1.begin AND t1.end) AND
-            (t2.offset_end BETWEEN t1.begin AND t1.end) AND
-            t2.id = t3.mp_id"(srcCovInfoTable, srcCovTable, mutationPointTable, mutationTable);
+            t0.file_id = t1.file_id AND
+            (t1.offset_begin BETWEEN t0.begin AND t0.end) AND
+            (t1.offset_end BETWEEN t0.begin AND t0.end) AND
+            t1.id = t2.mp_id"(srcCovTable, mutationPointTable, mutationTable);
 
         auto app = appender!(MutationStatusId[])();
         auto stmt = db.prepare(sql);

plugin/mutate/source/dextool/plugin/mutate/backend/test_mutant/coverage.d

@@ -7,7 +7,7 @@ module dextool.plugin.mutate.backend.test_mutant.coverage;
 
 import core.time : Duration;
 import logger = std.experimental.logger;
-import std.algorithm : map, filter, sort;
+import std.algorithm : map, filter, sort, canFind;
 import std.array : array, appender, empty;
 import std.exception : collectException;
 import std.stdio : File;
@@ -27,7 +27,7 @@ import dextool.plugin.mutate.backend.database : CovRegion, CoverageRegionId, Fil
 import dextool.plugin.mutate.backend.database : Database;
 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob;
 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner : TestRunner, TestResult;
-import dextool.plugin.mutate.backend.type : Mutation, Language;
+import dextool.plugin.mutate.backend.type : Mutation, Language, Offset;
 import dextool.plugin.mutate.type : ShellCommand, UserRuntime, CoverageRuntime;
 import dextool.plugin.mutate.config : ConfigCoverage;
 
@@ -44,6 +44,10 @@ struct CoverageDriver {
         bool hasRoot;
     }
 
+    static struct ImportJSON {
+
+    }
+
     static struct SaveOriginal {
     }
 
@@ -71,8 +75,8 @@ struct CoverageDriver {
     static struct Done {
     }
 
-    alias Fsm = my.fsm.Fsm!(None, Initialize, InitializeRoots, SaveOriginal,
-            Instrument, Compile, Run, SaveToDb, Restore, Done);
+    alias Fsm = my.fsm.Fsm!(None, ImportJSON, Initialize, InitializeRoots,
+            SaveOriginal, Instrument, Compile, Run, SaveToDb, Restore, Done);
 
     private {
         Fsm fsm;
@@ -108,6 +112,10 @@ struct CoverageDriver {
         Set!AbsolutePath roots;
 
         CoverageRuntime runtime;
+
+        bool importExists = false;
+
+        string metadataPath;
     }
 
     this(FilesysIO fio, Database* db, TestRunner* runner, ConfigCoverage conf,
@@ -119,6 +127,7 @@ struct CoverageDriver {
         this.buildCmdTimeout = buildCmdTimeout;
         this.log = conf.log;
         this.runtime = conf.runtime;
+        this.metadataPath = conf.metadataPath;
 
         foreach (a; conf.userRuntimeCtrl) {
             auto p = fio.toAbsoluteRoot(a.file);
@@ -131,7 +140,11 @@ struct CoverageDriver {
     }
 
     static void execute_(ref CoverageDriver self) @trusted {
-        self.fsm.next!((None a) => Initialize.init, (Initialize a) {
+        self.fsm.next!((None a) => ImportJSON.init, (ImportJSON a) {
+            if (self.importExists)
+                return fsm(Done.init);
+            return fsm(Initialize.init);
+        }, (Initialize a) {
             if (self.runtime == CoverageRuntime.inject)
                 return fsm(InitializeRoots.init);
             return fsm(SaveOriginal.init);
@@ -238,6 +251,114 @@ nothrow:
         }
     }
 
+    void opCall(ImportJSON data) {
+        import std.file : exists, readText;
+        import std.json : JSONValue, JSONOptions, parseJSON;
+        import std.string : startsWith;
+
+        if (metadataPath.length == 0) {
+            return;
+        } else if (!exists(metadataPath)) {
+            logger.error("File: " ~ metadataPath ~ " does not exist").collectException;
+            return;
+        }
+
+        string fileContent;
+        try {
+            fileContent = readText(metadataPath);
+        } catch (Exception e) {
+            logger.error("Unable to read file " ~ metadataPath).collectException;
+            return;
+        }
+
+        JSONValue jContent;
+        try {
+            jContent = parseJSON(fileContent, JSONOptions.doNotEscapeSlashes);
+        } catch (Exception e) {
+            logger.info(e.msg).collectException;
+            logger.error("Failed to parse filecontect of " ~ metadataPath ~ " into JSON")
+                .collectException;
+            return;
+        }
+
+        JSONValue objectData;
+        try {
+            objectData = jContent["coverage-info"];
+        } catch (Exception e) {
+            logger.info(e.msg).collectException;
+            logger.error("Object 'coverage-info' not found in file " ~ metadataPath)
+                .collectException;
+            return;
+        }
+
+        JSONValue filesObjectData;
+        try {
+            filesObjectData = objectData["file-keys"];
+        } catch (Exception e) {
+            logger.info(e.msg).collectException;
+            logger.error("Object 'file-keys' not found in file " ~ metadataPath).collectException;
+            return;
+        }
+        importExists = true;
+
+        try {
+            foreach (filePath; filesObjectData.arrayNoRef) {
+                auto fileData = objectData[filePath.str];
+
+                auto p = fio.toAbsoluteRoot(Path(filePath.str));
+                auto r = fio.getOutputDir();
+
+                if (p.toString.startsWith(r.toString)) {
+                    auto cov = fileData["coverage"];
+                    long[] covLines;
+                    foreach (line; cov.arrayNoRef) {
+                        covLines ~= line.integer;
+                    }
+                    auto noCov = fileData["no_coverage"];
+                    long[] noCovLines;
+                    foreach (line; noCov.arrayNoRef) {
+                        noCovLines ~= line.integer;
+                    }
+
+                    auto blob = fio.makeInput(p);
+
+                    Offset[] regions;
+                    bool[] statuses;
+                    int byteCounter = 0;
+                    int lineCounter = 1;
+                    uint lineStart = 0;
+                    uint lineEnd;
+                    foreach (b; blob.content) {
+                        if (b == '\n') {
+                            lineEnd = byteCounter;
+                            if (covLines.canFind(lineCounter) || noCovLines.canFind(lineCounter)) {
+                                Offset region;
+                                region.begin = lineStart;
+                                region.end = lineEnd;
+                                regions ~= region;
+                                if (covLines.canFind(lineCounter)) {
+                                    statuses ~= true;
+                                } else {
+                                    statuses ~= false;
+                                }
+                            }
+                            lineStart = byteCounter + 1;
+                            lineCounter++;
+                        }
+                        byteCounter++;
+                    }
+                    spinSql!(() @trusted {
+                        db.coverageApi.putCoverageMap(Path(filePath.str), regions, statuses);
+                    });
+                }
+            }
+        } catch (Exception e) {
+            logger.warning(e.msg).collectException;
+            logger.error("Faulty iteration").collectException;
+            return;
+        }
+    }
+
     void opCall(SaveOriginal data) {
         try {
             restore = regions.byKey.array;
@@ -352,7 +473,7 @@ nothrow:
         void save() @trusted {
             auto trans = db.transaction;
             foreach (a; data.covMap) {
-                db.coverageApi.putCoverageInfo(localId[a.id], a.status);
+                db.coverageApi.putCoverageStatus(localId[a.id], a.status);
             }
             db.coverageApi.updateCoverageTimeStamp;
             trans.commit;

plugin/mutate/source/dextool/plugin/mutate/backend/test_mutant/package.d

@@ -1611,7 +1611,7 @@ nothrow:
         auto covTimeStamp = spinSql!(() => db.coverageApi.getCoverageTimeStamp).orElse(
                 SysTime.init);
 
-        if (tracked < covTimeStamp) {
+        if (tracked < covTimeStamp && covConf.metadataPath.length == 0) {
             logger.info("Coverage information is up to date").collectException;
             return;
         } else {

plugin/mutate/source/dextool/plugin/mutate/config.d

@@ -322,6 +322,8 @@ struct ConfigCoverage {
     /// If the generated coverage files should be saved.
     bool log;
 
+    string metadataPath;
+
     /// allows a user to control exactly which files the coverage and schemata
     /// runtime is injected in.
     UserRuntime[] userRuntimeCtrl;

plugin/mutate/source/dextool/plugin/mutate/frontend/argparser.d

@@ -483,7 +483,7 @@ struct ArgParser {
                    "log-coverage", "write the instrumented coverage files to a separate file", &coverage.log,
                    "max-alive", "stop after NR alive mutants is found (only effective with -L or --diff-from-stdin)", &maxAlive,
                    "max-runtime", format("max time to run the mutation testing for (default: %s)", mutationTest.maxRuntime), &maxRuntime,
-                    "metadata", "prioritieses files that are sent by JSON", &mutationTest.metadataPath,
+                   "metadata", "prioritieses files that are sent by JSON", &mutationTest.metadataPath,
                    "m|mutant", "kind of mutation to test " ~ format("[%(%s|%)]", [EnumMembers!MutationKind]), &mutants,
                    "no-skipped", "do not skip mutants that are covered by others", &noSkip,
                    "order", "determine in what order mutants are chosen " ~ format("[%(%s|%)]", [EnumMembers!MutationOrder]), &mutationTest.mutationOrder,
@@ -528,6 +528,8 @@ struct ArgParser {
                 mutationTest.maxRuntime = parseDuration(maxRuntime);
             mutationTest.constraint = parseUserTestConstraint(testConstraint);
             mutationTest.useSkipMutant.get = !noSkip;
+
+            coverage.metadataPath = mutationTest.metadataPath;
         }
 
         void reportG(string[] args) {