Issue 10

README.md

@@ -55,7 +55,80 @@ This is consistent with the queries defined in the RDF data cube specification.
 
 ### SELECT queries
 
-SPARQL SELECT queries are considered to have failed if they return any matching solutions. Like ASK queries they should return bindings describing invalid resources. 
+SPARQL SELECT queries are considered to have failed if they return any matching solutions. Like ASK queries they should return bindings describing invalid resources.
+
+## Defining test suites
+
+A test suite defines a group of tests to be run. A test suite can be created from a single test file or a directory containing test files as shown in the
+examples above. A test suite can also be defined within an EDN file that lists the tests it contains. The minimal form of this EDN file is:
+
+```clojure
+{
+  :suite-name ["test1.sparql"
+               "dir/test2.sparql"]
+  :suite2     ["suite2/test3.sparql"]
+}
+```
+
+Each key in the top-level map defines a test suite and the corresponding value contains the suite definition. Each test definition in the associated
+list should be a path to a test file relative to the suite definition file. The type and name of each test is derived from the test file name. These
+can be stated explicitly by defining tests within a map:
+
+```clojure
+{
+  :suite-name [{:source "test1.sparql"
+                :type :sparql
+                :name "first"}
+               {:source "test2.sparql"
+                :name "second"}
+               {:source "test3.sparql"}
+               "dir/test4.sparql"]
+}
+```
+
+When defining test definitions explicitly, only the `:source` key is required, the type and name will be derived from the test file name if not
+provided. The two styles of defining tests can be combined within a test suite definition as defined above.
+
+### Combining test suites
+
+Test suites can selectively include test cases from other test suites:
+
+```clojure
+{
+  :suite1 ["test1.sparql"
+           "test2.sparql"]
+  :suite2 ["test3.sparql"]
+  :suite3 {:import [:suite1 :suite2]
+           :exclude [:suite1/test1]
+           :tests [{:source "test4.txt"
+                    :type :sparql}]}
+}
+```
+
+Test suites can import any number of other suites - this includes each test from the referenced suite into the importing suite. Any tests defined 
+in the imported suites can be selectively excluded by referencing them in the `:exclude` list. Each entry should contain a keyword of the form
+`:suite-name/test-name`. By default test names are the stem of the file name up to the file extension e.g. the test for file `"test1.sparql"`
+will be named `"test"`.
+
+Test suite extensions must be acyclic e.g. `:suite1` importing `:suite2` which in turn imports `:suite1` is an error.
+An error will be raised if any suite listed within an extension list is not defined, but suites do not need to be defined within the
+same suite file. For example given two test files:
+
+#### suite1.edn
+```clojure
+{:suite1 ["test1.sparql"]}
+```
+
+#### suite2.edn
+```clojure
+{:suite2 {:import [:suite1]
+          :tests ["test2.sparql"]}}
+```
+
+this is valid as long as `suite1.edn` is provided as a suite whenever `suite2.edn` is required e.g.
+
+    java -jar rdf-validator-standalone.jar --endpoint data.ttl --suite suite1.edn --suite suite2.edn
+
 
 ## License
 

project.clj

@@ -7,6 +7,7 @@
   :dependencies [[org.clojure/clojure "1.8.0"]
                  [org.clojure/tools.cli "0.3.7"]
                  [org.clojure/tools.logging "0.4.1"]
+                 [com.stuartsierra/dependency "0.2.0"]
                  [grafter "0.11.5"]
                  [org.apache.jena/apache-jena-libs "3.8.0" :extension "pom"]
                  [selmer "1.12.0"]
@@ -18,4 +19,5 @@
                  [org.apache.logging.log4j/log4j-slf4j-impl "2.11.0"]]
   :main ^:skip-aot rdf-validator.core
   :target-path "target/%s"
-  :profiles {:uberjar {:aot :all}})
+  :profiles {:uberjar {:aot :all}
+             :dev {:resource-paths ["test/resources"]}})

src/rdf_validator/core.clj

@@ -2,77 +2,23 @@
   (:gen-class)
   (:require [clojure.java.io :as io]
             [clojure.tools.cli :as cli]
-            [grafter.rdf :as rdf]
-            [grafter.rdf.repository :as repo]
-            [clojure.string :as string]
             [rdf-validator.endpoint :as endpoint]
             [rdf-validator.query :as query]
+            [rdf-validator.reporting :as reporting]
+            [rdf-validator.test-cases :as tc]
             [selmer.parser :as selmer]
             [selmer.util :refer [without-escaping set-missing-value-formatter!]]
-            [clojure.tools.logging :as log]
             [clojure.edn :as edn])
-  (:import [java.net URI URISyntaxException]
+  (:import [java.net URI]
            [org.apache.jena.query QueryFactory Syntax]
            [java.io File]))
 
-(defn file->repository [^File f]
-  (if (.isDirectory f)
-    (let [r (repo/sail-repo)]
-      (log/info "Creating repository from directory: " (.getAbsolutePath f))
-      (doseq [df (.listFiles f)]
-        (rdf/add r (rdf/statements df)))
-      r)
-    (do
-      (log/info "Creating repository from file: " (.getAbsolutePath f))
-      (repo/fixture-repo f))))
-
-(defmulti uri->repository (fn [^URI uri] (some-> (.getScheme uri) keyword)))
-
-(defn- create-sparql-repo [uri]
-  (log/info "Creating SPARQL repository: " (str uri))
-  (repo/sparql-repo (str uri)))
-
-(defmethod uri->repository :http [uri]
-  (create-sparql-repo uri))
-
-(defmethod uri->repository :https [uri]
-  (create-sparql-repo uri))
-
-(defmethod uri->repository :file [uri]
-  (file->repository (io/file uri)))
-
-(defmethod uri->repository :default [uri]
-  (file->repository (io/file (str uri))))
-
-(defn parse-endpoint [endpoint-str]
-  (try
-    (uri->repository (URI. endpoint-str))
-    (catch URISyntaxException ex
-      (file->repository (io/file endpoint-str)))))
-
 (defn- conj-in [m k v]
   (update-in m [k] conj v))
+
 (defn parse-variables-file [f]
   (edn/read-string (slurp f)))
 
-(def cli-options
-  [["-s" "--suite SUITE" "Test suite file or directory"
-    :default []
-    :parse-fn io/file
-    :validate [(fn [f]
-                 ;;TODO: check file exists
-                 true) "File does not exist"]
-    :assoc-fn conj-in]
-   ["-e" "--endpoint ENDPOINT" "SPARQL data endpoint to validate"
-    :parse-fn parse-endpoint]
-   ["-g" "--graph GRAPH" "Graph to include in the RDF dataset"
-    :default []
-    :parse-fn #(URI. %)
-    :assoc-fn conj-in]
-   ["-v" "--variables FILE" "EDN file containing query variables"
-    :parse-fn parse-variables-file
-    :default {}]])
-
 (defn- usage [summary]
   (println "Usage:")
   (println summary))
@@ -93,93 +39,91 @@
   (let [query-string (slurp f)]
     (without-escaping (selmer/render query-string variables))))
 
-(defn load-test-case [^File f query-variables]
+(defn find-test-files [suites]
+  (mapcat (fn [^File f]
+            (if (.isDirectory f)
+              (find-test-files (.listFiles f))
+              [f]))
+          suites))
+
+(defn run-sparql-ask-test [{:keys [source-file query-string]} endpoint]
+  (let [pquery (endpoint/prepare-query endpoint query-string)
+        failed (query/execute pquery)]
+    {:source-file source-file
+     :result      (if failed :failed :passed)
+     :errors      (if failed ["ASK query returned true"] [])}))
+
+(defn run-sparql-select-test [{:keys [source-file query-string]} endpoint]
+  (let [pquery (endpoint/prepare-query endpoint query-string)
+        results (query/execute pquery)
+        failed (pos? (count results))]
+    {:source-file source-file
+     :result      (if failed :failed :passed)
+     :errors      (mapv str results)}))
+
+(defn run-test-case [{f :source :as test-case} query-variables endpoint]
   (try
     (let [^String sparql-str (load-sparql-template f query-variables)
           query (QueryFactory/create sparql-str Syntax/syntaxSPARQL_11)
-          type (cond
-                 (.isAskType query) :sparql-ask
-                 (.isSelectType query) :sparql-select
-                 :else :sparql-ignored)]
-      {:source-file  f
-       :type         type
-       :query-string sparql-str})
+          test {:source-file f :query-string sparql-str}]
+      (cond
+        (.isAskType query) (run-sparql-ask-test test endpoint)
+        (.isSelectType query) (run-sparql-select-test test endpoint)
+        :else {:source-file f
+               :result :ignored
+               :errors []}))
     (catch Exception ex
       {:source-file f
-       :type :invalid
-       :exception ex})))
-
-(defn load-test-cases [^File f query-variables]
-  (if (.isDirectory f)
-    (mapcat (fn [cf] (load-test-cases cf query-variables)) (.listFiles f))
-    [(load-test-case f query-variables)]))
-
-(defmulti run-test-case (fn [test-case endpoint] (:type test-case)))
-
-(defmethod run-test-case :sparql-ask [{:keys [query-string source-file] :as test-case} endpoint]
-  (try
-    (let [pquery (endpoint/prepare-query endpoint query-string)
-          failed (query/execute pquery)]
-      {:source-file source-file
-       :result      (if failed :failed :passed)
-       :errors      (if failed ["ASK query returned true"] [])})
-    (catch Exception ex
-      {:source-file source-file
-       :result :errored
-       :errors [(.getMessage ex)]})))
-
-(defmethod run-test-case :sparql-select [{:keys [query-string source-file] :as test-case} endpoint]
-  (try
-    (let [pquery (endpoint/prepare-query endpoint query-string)
-          results (query/execute pquery)
-          failed (pos? (count results))]
-      {:source-file source-file
-       :result      (if failed :failed :passed)
-       :errors      (mapv str results)})
-    (catch Exception ex
-      {:source-file source-file
        :result :errored
        :errors [(.getMessage ex)]})))
 
-(defmethod run-test-case :sparql-ignored [{:keys [source-file]} _endpoint]
-  {:source-file source-file
-   :result :ignored
-   :errors []})
-
-(defmethod run-test-case :invalid [{:keys [source-file ^Throwable exception]} _endpoint]
-  {:source-file source-file
-   :result :errored
-   :errors [(.getMessage exception)]})
-
-(defn display-test-result [{:keys [number ^File source-file result errors] :as test-result}]
-  (println (format "%d %s: %s" number (.getAbsolutePath source-file) (string/upper-case (name result))))
-  (doseq [error errors]
-    (println (format "\t%s" error)))
-  (when (pos? (count errors))
-    (println)))
-
-(defn run-test-cases [test-cases endpoint]
-  (reduce (fn [summary [test-index test-case]]
-            (let [{:keys [result] :as test-result} (run-test-case test-case endpoint)]
-              (display-test-result (assoc test-result :number (inc test-index)))
-              (update summary result inc)))
-          {:failed 0 :passed 0 :errored 0 :ignored 0}
-          (map-indexed vector test-cases)))
+(defn run-test-cases [test-cases query-variables endpoint reporter]
+  (let [summary (reduce (fn [summary [test-index test-case]]
+                          (let [{:keys [result] :as test-result} (run-test-case test-case query-variables endpoint)]
+                            (reporting/report-test-result! reporter (assoc test-result :number (inc test-index)))
+                            (update summary result inc)))
+                        {:failed 0 :passed 0 :errored 0 :ignored 0}
+                        (map-indexed vector test-cases))]
+    (reporting/report-test-summary! reporter summary)
+    summary))
 
 (defn- create-endpoint [{:keys [endpoint graph] :as options}]
   (endpoint/create-endpoint endpoint graph))
 
+(def cli-options
+  [["-s" "--suite SUITE" "Test suite file or directory"
+    :default []
+    :parse-fn io/file
+    :validate [(fn [f]
+                 ;;TODO: check file exists
+                 true) "File does not exist"]
+    :assoc-fn conj-in]
+   ["-e" "--endpoint ENDPOINT" "SPARQL data endpoint to validate"
+    :parse-fn endpoint/parse-repository]
+   ["-g" "--graph GRAPH" "Graph to include in the RDF dataset"
+    :default []
+    :parse-fn #(URI. %)
+    :assoc-fn conj-in]
+   ["-v" "--variables FILE" "EDN file containing query variables"
+    :parse-fn parse-variables-file
+    :default {}]])
+
 (defn -main
   [& args]
   (let [{:keys [errors options] :as result} (cli/parse-opts args cli-options)]
     (if (nil? errors)
-      (let [suites (:suite options)
-            endpoint (create-endpoint options)
-            query-variables (:variables options)
-            test-cases (mapcat (fn [f] (load-test-cases f query-variables)) suites)
-            {:keys [passed failed errored ignored]} (run-test-cases test-cases endpoint)]
-        (println)
-        (println (format "Passed %d Failed %d Errored %d Ignored %d" passed failed errored ignored))
-        (System/exit (+ failed errored)))
+      (try
+        (let [suite-files (:suite options)
+              endpoint (create-endpoint options)
+              query-variables (:variables options)
+              suites (tc/resolve-test-suites suite-files)
+              test-cases (tc/suite-tests suites)
+              test-reporter (reporting/->ConsoleTestReporter)
+              {:keys [failed errored] :as test-summary} (run-test-cases test-cases query-variables endpoint test-reporter)]
+          (System/exit (+ failed errored)))
+        (catch Exception ex
+          (binding [*out* *err*]
+            (println (.getMessage ex))
+            (System/exit 1))))
       (do (invalid-args result)
           (System/exit 1)))))

src/rdf_validator/endpoint.clj

@@ -1,5 +1,47 @@
 (ns rdf-validator.endpoint
-  (:require [grafter.rdf.repository :as repo]))
+  (:require [grafter.rdf.repository :as repo]
+            [grafter.rdf :as rdf]
+            [clojure.java.io :as io]
+            [clojure.tools.logging :as log])
+  (:import [java.net URISyntaxException URI]
+           [java.io File]))
+
+(defn file->repository [^File f]
+  (if (.isDirectory f)
+    (let [r (repo/sail-repo)]
+      (log/info "Creating repository from directory: " (.getAbsolutePath f))
+      (doseq [df (.listFiles f)]
+        (rdf/add r (rdf/statements df)))
+      r)
+    (do
+      (log/info "Creating repository from file: " (.getAbsolutePath f))
+      (repo/fixture-repo f))))
+
+(defmulti uri->repository (fn [^URI uri] (some-> (.getScheme uri) keyword)))
+
+(defn- create-sparql-repo [uri]
+  (log/info "Creating SPARQL repository: " (str uri))
+  (repo/sparql-repo (str uri)))
+
+(defmethod uri->repository :http [uri]
+  (create-sparql-repo uri))
+
+(defmethod uri->repository :https [uri]
+  (create-sparql-repo uri))
+
+(defmethod uri->repository :file [uri]
+  (file->repository (io/file uri)))
+
+(defmethod uri->repository :default [uri]
+  (file->repository (io/file (str uri))))
+
+(defn parse-repository
+  "Parses a sesame repository instance from a string representation"
+  [repository-str]
+  (try
+    (uri->repository (URI. repository-str))
+    (catch URISyntaxException ex
+      (file->repository (io/file repository-str)))))
 
 (defn- create-dataset [graphs]
   (if-let [restriction (seq (map str graphs))]