From 49b29e0487784c94c36580c3dd3fcd7c3c138ca2 Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Tue, 28 May 2019 11:41:53 -0600 Subject: [PATCH 01/10] Configure midje for vim-iced-kaocha --- project.clj | 4 +++- tests.edn | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 tests.edn diff --git a/project.clj b/project.clj index 9ea93661..b5acd215 100644 --- a/project.clj +++ b/project.clj @@ -59,7 +59,9 @@ :dev [:profiles/dev {:plugins [[lein-midje "3.2.1"]] :dependencies [[midje "1.9.6"] - [nubank/matcher-combinators "0.9.0"]]}] + [lambdaisland/kaocha-midje "0.0-5" + :exclusions [midje/midje]] + [nubank/matcher-combinators "0.9.0"]]}] :test {:resource-paths ["test/resources"] :env {:yb-adapters-freenode-type "irc" diff --git a/tests.edn b/tests.edn new file mode 100644 index 00000000..c0cce960 --- /dev/null +++ b/tests.edn @@ -0,0 +1,3 @@ +#kaocha/v1 +{:tests [{:id :unit + :type :kaocha.type/midje}]} From b4a8bc86662c601e0e5adc5edc8cffa54b4d562b Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Thu, 30 May 2019 13:24:20 -0600 Subject: [PATCH 02/10] Add adapters query --- resources/graphql-schema.edn | 28 ++++------ src/yetibot/core/webapp/resolvers.clj | 32 +---------- src/yetibot/core/webapp/resolvers/adapter.clj | 0 src/yetibot/core/webapp/resolvers/history.clj | 56 +++++++++++++++++++ src/yetibot/core/webapp/routes/graphql.clj | 50 +++++++++-------- tests.edn | 4 +- 6 files changed, 98 insertions(+), 72 deletions(-) create mode 100644 src/yetibot/core/webapp/resolvers/adapter.clj create mode 100644 src/yetibot/core/webapp/resolvers/history.clj diff --git a/resources/graphql-schema.edn b/resources/graphql-schema.edn index 011e1869..0c63d848 100644 --- a/resources/graphql-schema.edn +++ b/resources/graphql-schema.edn @@ -1,11 +1,4 @@ -{:objects {:adapter {:fields {:uuid {:type String} - :platform {:type String} - :is_connected {:type Boolean} - :connection_latency {:type Int} - :connection_last_active_timestamp {:type String} - }} - - :history {:fields +{:objects {:history {:fields {:chat_source_adapter {:type String} :chat_source_room {:type String} :user {:type :user @@ -67,12 +60,13 @@ :cmd {:type String}}} ;; TODO list channels by adapter - :adapter_channel {:fields - {:adapter {:type String} - :channel {:type String}}} + :adapter {:fields + {:chat_source_adapter {:type String} + :channels {:type (list :channel)}}} :channel {:fields - {:name {:type String}}} + {:chat_source_adapter {:type String} + :chat_source_room {:type String}}} :karma {:fields {:user_id {:type String} @@ -88,13 +82,17 @@ :args {:expr {:type String}} :resolve :eval} + ;; Deprecated: use adapters + ;; :channels {:type (list :channel) + ;; :resolve :channels} + :adapters {:type (list :adapter) :resolve :adapters} :history {:type (list :history) :resolve :history :args {:limit {:type Int - :default-value 20} + :default-value 50} :offset {:type Int :default-value 0} :chat_source_adapter {:type String} @@ -115,9 +113,6 @@ :resolve :history_item :args {:id {:type Int}}} - :channels {:type (list :channel) - :resolve :channels} - :users {:type (list :user) :resolve :users} @@ -146,4 +141,5 @@ :args {:report {:type :karma_report} :limit {:type Int :default-value 10}}} + }} diff --git a/src/yetibot/core/webapp/resolvers.clj b/src/yetibot/core/webapp/resolvers.clj index 693830fe..14550a5d 100644 --- a/src/yetibot/core/webapp/resolvers.clj +++ b/src/yetibot/core/webapp/resolvers.clj @@ -3,7 +3,6 @@ [yetibot.core.models.users :as users] [yetibot.core.chat :as chat] [yetibot.core.models.channel :as channel] - [yetibot.core.webapp.resolvers.stats :as stats] [cuerdas.core :refer [kebab snake]] [yetibot.core.adapters.adapter :as adapter] [yetibot.core.db.history :as history] @@ -38,6 +37,7 @@ "echo hi") ) + (defn adapters-resolver [context {:keys [] :as args} value] (->> @@ -50,34 +50,6 @@ :connection_last_active_timestamp (adapter/connection-last-active-timestamp %))))) -(defn history-resolver - [context - {:keys [offset limit chat_source_room chat_source_adapter commands_only - yetibot_only search_query user_filter channel_filter] - :as args} - value] - (info "history resolver. args" args) - (let [where-map (merge {"is_private" false} - (when commands_only {"is_command" true}) - (when yetibot_only {"is_yetibot" true})) - where-clause (when search_query - {:where/clause - "to_tsvector(body) @@ plainto_tsquery(?)" - :where/args [search_query]}) - ] - (history/query (merge {:query/identifiers identity - :where/map where-map - :limit/clause limit - :offset/clause offset - :order/clause "created_at DESC"} - where-clause)))) - -(defn history-item-resolver - [_ {:keys [id] :as args} _] - (let [where-map {"id" id}] - (first - (history/query (merge {:query/identifiers identity - :where/map where-map}))))) (defn channels-resolver [context args value] @@ -87,8 +59,6 @@ (map #(hash-map :name %) (chat/channels)))) (adapter/active-adapters))) -(def stats-resolver (partial stats/stats-resolver)) - (defn users-resolver [context {:keys [] :as args} value] (map (fn [{:keys [username active? id last-active]}] diff --git a/src/yetibot/core/webapp/resolvers/adapter.clj b/src/yetibot/core/webapp/resolvers/adapter.clj new file mode 100644 index 00000000..e69de29b diff --git a/src/yetibot/core/webapp/resolvers/history.clj b/src/yetibot/core/webapp/resolvers/history.clj new file mode 100644 index 00000000..e7869c72 --- /dev/null +++ b/src/yetibot/core/webapp/resolvers/history.clj @@ -0,0 +1,56 @@ +(ns yetibot.core.webapp.resolvers.history + (:require + [yetibot.core.db.history :refer [query]] + [yetibot.core.models.history :as history] + [yetibot.core.db.alias :as db.alias] + [yetibot.core.db.observe :as db.observe] + [yetibot.core.db.cron :as db.cron] + [yetibot.core.commands.uptime :as uptime] + [yetibot.core.adapters.adapter :as adapter] + [yetibot.core.models.users :as users] + [com.walmartlabs.lacinia.executor :refer [selections-seq selects-field?]] + [taoensso.timbre :refer [error debug info color-str]])) + +(defn history-resolver + [context + {:keys [offset limit chat_source_room chat_source_adapter commands_only + yetibot_only search_query user_filter channel_filter] + :as args} + value] + (info "history resolver. args" args) + (let [where-map (merge {"is_private" false} + (when commands_only {"is_command" true}) + (when yetibot_only {"is_yetibot" true})) + where-clause (when search_query + {:where/clause + "to_tsvector(body) @@ plainto_tsquery(?)" + :where/args [search_query]})] + (query (merge {:query/identifiers identity + :where/map where-map + :limit/clause limit + :offset/clause offset + :order/clause "created_at DESC"} + where-clause)))) + +(defn history-item-resolver + [_ {:keys [id] :as args} _] + (let [where-map {"id" id}] + (first + (query (merge {:query/identifiers identity + :where/map where-map}))))) + +(defn adapters-resolver + [_ _ _] + (for [[k v] (group-by + :chat_source_adapter + (query + {:query/identifiers identity + :where/map {:is_private false} + :select/clause + "chat_source_adapter, chat_source_room" + :group/clause + "chat_source_adapter, chat_source_room" + :order/clause + "chat_source_adapter, chat_source_room"}))] + {:chat_source_adapter k + :channels v})) diff --git a/src/yetibot/core/webapp/routes/graphql.clj b/src/yetibot/core/webapp/routes/graphql.clj index 2a389fa5..abb1da2b 100644 --- a/src/yetibot/core/webapp/routes/graphql.clj +++ b/src/yetibot/core/webapp/routes/graphql.clj @@ -1,35 +1,37 @@ (ns yetibot.core.webapp.routes.graphql (:require - [clojure.walk :refer [keywordize-keys]] - [clojure.edn :as edn] - [clojure.data.json :as json] - [clojure.java.io :as io] - [com.walmartlabs.lacinia :refer [execute]] - [com.walmartlabs.lacinia.schema :as lacina.schema] - [com.walmartlabs.lacinia.util :as lacina.util] - [compojure.core :refer [defroutes POST OPTIONS]] - [taoensso.timbre :refer [error debug info color-str]] - [yetibot.core.webapp.resolvers :as resolvers])) + [clojure.walk :refer [keywordize-keys]] + [clojure.edn :as edn] + [clojure.data.json :as json] + [clojure.java.io :as io] + [com.walmartlabs.lacinia :refer [execute]] + [com.walmartlabs.lacinia.schema :as lacinia.schema] + [com.walmartlabs.lacinia.util :as lacinia.util] + [compojure.core :refer [defroutes POST OPTIONS]] + [taoensso.timbre :refer [error debug info color-str]] + [yetibot.core.webapp.resolvers :as resolvers] + [yetibot.core.webapp.resolvers.stats :refer [stats-resolver]] + [yetibot.core.webapp.resolvers.history :refer [adapters-resolver + history-resolver + history-item-resolver]])) (defn load-schema! [] (-> (io/resource "graphql-schema.edn") slurp edn/read-string - (lacina.util/attach-resolvers {:eval resolvers/eval-resolver - :adapters resolvers/adapters-resolver - :history resolvers/history-resolver - :history_item resolvers/history-item-resolver - :users resolvers/users-resolver - :user resolvers/user-resolver - :stats resolvers/stats-resolver - :aliases resolvers/aliases-resolver - :observers resolvers/observers-resolver - :crons resolvers/crons-resolver - :channels resolvers/channels-resolver - :karmas resolvers/karmas-resolver - }) - lacina.schema/compile)) + (lacinia.util/attach-resolvers {:eval resolvers/eval-resolver + :history history-resolver + :history_item history-item-resolver + :users resolvers/users-resolver + :user resolvers/user-resolver + :stats stats-resolver + :aliases resolvers/aliases-resolver + :observers resolvers/observers-resolver + :crons resolvers/crons-resolver + :adapters adapters-resolver + :karmas resolvers/karmas-resolver}) + lacinia.schema/compile)) ;; note this is not reloadable (def schema (delay (load-schema!))) diff --git a/tests.edn b/tests.edn index c0cce960..d6fac928 100644 --- a/tests.edn +++ b/tests.edn @@ -1,3 +1,5 @@ #kaocha/v1 -{:tests [{:id :unit +{:tests [{:source-paths ["src"] + :ns-patterns [".*"] + :test-paths ["test"] :type :kaocha.type/midje}]} From 5e460daa0564450a41350995c94ee57c3c573095 Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Mon, 3 Jun 2019 11:48:21 -0600 Subject: [PATCH 03/10] Add latest history graphql WIP --- resources/graphql-schema.edn | 310 ++++++++++-------- src/yetibot/core/commands/history.clj | 2 +- src/yetibot/core/webapp/routes/graphql.clj | 9 +- .../core/test/webapp/routes/graphql.clj | 79 +++-- 4 files changed, 223 insertions(+), 177 deletions(-) diff --git a/resources/graphql-schema.edn b/resources/graphql-schema.edn index 0c63d848..5512b809 100644 --- a/resources/graphql-schema.edn +++ b/resources/graphql-schema.edn @@ -1,145 +1,177 @@ -{:objects {:history {:fields - {:chat_source_adapter {:type String} - :chat_source_room {:type String} - :user {:type :user - :resolve :user} - :user_id {:type String} - :user_name {:type String} - :body {:type String} - :is_command {:type Boolean} - :is_yetibot {:type Boolean} - :id {:type Int} - :command {:type String} - :correlation_id {:type String} - :created_at {:type String}}} - - :stats {:fields - {:uptime {:type String} - :adapter_count {:type Int} - :user_count {:type Int} - :history_count {:type Int} - :history_count_today {:type Int} - :command_count_today {:type Int} - :command_count {:type Int} - :alias_count {:type Int} - :observer_count {:type Int} - :cron_count {:type Int} - }} - - :user {:fields - {:username {:type String} - :is_active {:type Boolean} - :id {:type String} - :last_active {:type String} - :karma {:type Int}}} - - :alias {:fields - {:id {:type Int} - :user_id {:type String} - :cmd_name {:type String} - :cmd {:type String} - :created_at {:type String}}} - - :observer {:fields - {:id {:type Int} - :user_id {:type String} - :pattern {:type String} - :user_pattern {:type String} - :channel_pattern {:type String} - :event_type {:type String} - :cmd {:type String} - :created_at {:type String}}} - - :cron {:fields - {:id {:type Int} - :created_at {:type String} - :chat_source_adapter {:type String} - :chat_source_room {:type String} - :user_id {:type String} - :schedule {:type String} - :cmd {:type String}}} - - ;; TODO list channels by adapter - :adapter {:fields - {:chat_source_adapter {:type String} - :channels {:type (list :channel)}}} - - :channel {:fields - {:chat_source_adapter {:type String} - :chat_source_room {:type String}}} - - :karma {:fields - {:user_id {:type String} - :score {:type Int}}} - } +{:objects + {:history {:fields + {:chat_source_adapter {:type String} + :chat_source_room {:type String} + :user {:type :user + :resolve :user} + :user_id {:type String} + :user_name {:type String} + :body {:type String} + :is_command {:type Boolean} + :is_yetibot {:type Boolean} + :id {:type Int} + :command {:type String} + :correlation_id {:type String} + :created_at {:type String}}} + + :page_info {:fields + {:total_results {:type Int} + :next_page_cursor {:type String} + :has_next_page {:type Boolean}}} + + :paged_history {:fields + {:page_info {:type :page_info} + :history {:type (list :history)}}} + + ;; { + ;; hero { + ;; name + ;; friendsConnection(first:2 after:"Y3Vyc29yMQ==") { + ;; totalCount + ;; edges { + ;; node { + ;; name + ;; } + ;; cursor + ;; } + ;; pageInfo { + ;; endCursor + ;; hasNextPage + ;; } + ;; } + ;; } + ;; } + + :stats {:fields + {:uptime {:type String} + :adapter_count {:type Int} + :user_count {:type Int} + :history_count {:type Int} + :history_count_today {:type Int} + :command_count_today {:type Int} + :command_count {:type Int} + :alias_count {:type Int} + :observer_count {:type Int} + :cron_count {:type Int} + }} + + :user {:fields + {:username {:type String} + :is_active {:type Boolean} + :id {:type String} + :last_active {:type String} + :karma {:type Int}}} + + :alias {:fields + {:id {:type Int} + :user_id {:type String} + :cmd_name {:type String} + :cmd {:type String} + :created_at {:type String}}} + + :observer {:fields + {:id {:type Int} + :user_id {:type String} + :pattern {:type String} + :user_pattern {:type String} + :channel_pattern {:type String} + :event_type {:type String} + :cmd {:type String} + :created_at {:type String}}} + + :cron {:fields + {:id {:type Int} + :created_at {:type String} + :chat_source_adapter {:type String} + :chat_source_room {:type String} + :user_id {:type String} + :schedule {:type String} + :cmd {:type String}}} + + ;; TODO add adapter info back + :adapter {:fields + {:chat_source_adapter {:type String} + :channels {:type (list :channel)}}} + + :channel {:fields + {:chat_source_adapter {:type String} + :chat_source_room {:type String}}} + + :karma {:fields + {:user_id {:type String} + :score {:type Int}}}} :enums {:karma_report {:values [{:enum-value :SCORES :description "Users with the most amassed karma"} {:enum-value :GIVERS :description "User who have bestored the most karma"}]}} - :queries {:eval {:type (list String) - :args {:expr {:type String}} - :resolve :eval} - - ;; Deprecated: use adapters - ;; :channels {:type (list :channel) - ;; :resolve :channels} - - :adapters {:type (list :adapter) - :resolve :adapters} - - :history {:type (list :history) - :resolve :history - :args {:limit {:type Int - :default-value 50} - :offset {:type Int - :default-value 0} - :chat_source_adapter {:type String} - :chat_source_room {:type String} - :commands_only {:type Boolean - :default-value false} - :yetibot_only {:type Boolean - :default-value false} - :search_query {:type String - :default-value nil} - :user_filter {:type String - :default-value nil} - :channel_filter {:type String - :default-value nil} - }} - - :history_item {:type :history - :resolve :history_item - :args {:id {:type Int}}} - - :users {:type (list :user) - :resolve :users} - - :user {:type :user - :resolve :user - :args {:id {:type String}}} - - :stats {:type :stats - :resolve :stats - :args {:timezone_offset_hours {:type Int - ;; default UTC - :default-value 0} - }} - - :aliases {:type (list :alias) - :resolve :aliases} - - :observers {:type (list :observer) - :resolve :observers} - - :crons {:type (list :cron) - :resolve :crons} - - :karmas {:type (list :karma) - :resolve :karmas - :args {:report {:type :karma_report} - :limit {:type Int - :default-value 10}}} - - }} + :queries + {:eval {:type (list String) + :args {:expr {:type String}} + :resolve :eval} + + :adapters {:type (list :adapter) + :resolve :adapters} + + :history {:type :paged_history + :resolve :history + :args {;; pagination + :first {:type Int + :default-value 50} + :cursor {:type Int + :default-value nil} + + ;; filtering + :include_history_commands {:type Boolean + :default-value false} + :exclude_yetibot {:type Boolean + :default-valule false} + :exclude_commands {:type Boolean + :default-valule false} + :exclude_non_commands {:type Boolean + :default-valule false} + + :search_query {:type String + :default-value nil} + :users_filter {:type (list String) + :default-value nil} + :adapters_filter {:type (list String) + :default-value nil} + :channels_filter {:type (list String) + :default-value nil}}} + + :history_item {:type :history + :resolve :history_item + :args {:id {:type Int}}} + + :users {:type (list :user) + :resolve :users} + + :user {:type :user + :resolve :user + :args {:id {:type String}}} + + :stats {:type :stats + :resolve :stats + :args {:timezone_offset_hours {:type Int + ;; default UTC + :default-value 0} + }} + + :aliases {:type (list :alias) + :resolve :aliases} + + :observers {:type (list :observer) + :resolve :observers} + + :crons {:type (list :cron) + :resolve :crons} + + :karmas {:type (list :karma) + :resolve :karmas + :args {:report {:type :karma_report} + :limit {:type Int + :default-value 10}}} + + }} diff --git a/src/yetibot/core/commands/history.clj b/src/yetibot/core/commands/history.clj index fc993c1e..e8783cc0 100644 --- a/src/yetibot/core/commands/history.clj +++ b/src/yetibot/core/commands/history.clj @@ -120,7 +120,7 @@ few collection commands where it will bake the expression into a single SQL query instead of trying to naively evaluate in memory: - history | grep - uses Postgres' ~ operator to search + history | grep - search history using Postgres ~ operator history | tail [] - uses LIMIT n and ORDER_BY history | head [] - uses LIMIT n and ORDER_BY history | random - uses LIMIT 1 Postgres' ORDER_BY random() diff --git a/src/yetibot/core/webapp/routes/graphql.clj b/src/yetibot/core/webapp/routes/graphql.clj index abb1da2b..d9d974d5 100644 --- a/src/yetibot/core/webapp/routes/graphql.clj +++ b/src/yetibot/core/webapp/routes/graphql.clj @@ -41,10 +41,11 @@ (def context {}) (defn graphql - [query variables] - (let [keyword-vars (keywordize-keys variables)] - (debug "graphql" {:query query :variables keyword-vars}) - (execute @schema query keyword-vars context))) + ([query] (graphql query {})) + ([query variables] + (let [keyword-vars (keywordize-keys variables)] + (debug "graphql" {:query query :variables keyword-vars}) + (execute @schema query keyword-vars context)))) (defroutes graphql-routes (POST "/graphql" [query variables] (json/write-str (graphql query variables)))) diff --git a/test/yetibot/core/test/webapp/routes/graphql.clj b/test/yetibot/core/test/webapp/routes/graphql.clj index 496c75a4..603992bb 100644 --- a/test/yetibot/core/test/webapp/routes/graphql.clj +++ b/test/yetibot/core/test/webapp/routes/graphql.clj @@ -1,37 +1,50 @@ (ns yetibot.core.test.webapp.routes.graphql (:require - [yetibot.core.webapp.routes.graphql :refer [graphql]] - [clojure.test :refer :all])) + [midje.sweet :refer [fact => contains has-prefix just]] + [yetibot.core.midje :refer [value data error]] + [yetibot.core.webapp.routes.graphql :refer [graphql]])) -(deftest graphql-test - (testing "Simple graphql query" - (is - (= (-> (graphql - "{eval(expr: \"echo foo | echo bar\")}" - {}) - :data - first) - [:eval ["bar foo"]]))) +(fact "graphql can evaluate a simple expression" + (graphql "{eval(expr: \"echo foo | echo bar\")}") + => {:data {:eval ["bar foo"]}}) - (testing "Query with Variables" - (is - (not - (:errors - (graphql - "query stats($timezone_offset_hours: Int!) { - stats(timezone_offset_hours: $timezone_offset_hours) { - uptime - adapter_count - user_count - command_count_today - command_count - history_count - history_count_today - alias_count - observer_count - cron_count - } - }" - {"timezone_offset_hours" 6} - ))))) - ) +(fact "graphql can accept variables" + (graphql + "query stats($timezone_offset_hours: Int!) { + stats(timezone_offset_hours: $timezone_offset_hours) { + uptime + adapter_count + user_count + command_count_today + command_count + history_count + history_count_today + alias_count + observer_count + cron_count + } + }" + {"timezone_offset_hours" 6}) =not=> (contains {:errors coll?})) + + +(fact "graphql can filter history" + (graphql " + history(limit: 30, + offset: 0, + commands_only: $commands_only, + yetibot_only: $yetibot_only, + search_query: $search_query + ) { + id + chat_source_adapter + chat_source_room + command + correlation_id + created_at + user_name + is_command + is_yetibot + body + user_id + user_name + }") From 1640716f89dedd6521b45cec99086b710212b2e3 Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Fri, 14 Jun 2019 07:24:36 -0600 Subject: [PATCH 04/10] Move parameterized history query generation to history model --- resources/graphql-schema.edn | 10 +- src/yetibot/core/commands/history.clj | 135 ++++++------------ src/yetibot/core/db/util.clj | 6 + src/yetibot/core/models/history.clj | 102 +++++++++++-- src/yetibot/core/webapp/resolvers/history.clj | 67 ++++++--- test/yetibot/core/test/commands/history.clj | 44 +++--- test/yetibot/core/test/db/util.clj | 5 + .../core/test/webapp/routes/graphql.clj | 39 ++--- 8 files changed, 241 insertions(+), 167 deletions(-) diff --git a/resources/graphql-schema.edn b/resources/graphql-schema.edn index 5512b809..ffa4e542 100644 --- a/resources/graphql-schema.edn +++ b/resources/graphql-schema.edn @@ -119,7 +119,7 @@ :args {;; pagination :first {:type Int :default-value 50} - :cursor {:type Int + :cursor {:type String :default-value nil} ;; filtering @@ -139,7 +139,13 @@ :adapters_filter {:type (list String) :default-value nil} :channels_filter {:type (list String) - :default-value nil}}} + :default-value nil} + ;; date filtering + :until_datetime {:type String + :default-value nil} + :since_datetime {:type String + :default-value nil} + }} :history_item {:type :history :resolve :history_item diff --git a/src/yetibot/core/commands/history.clj b/src/yetibot/core/commands/history.clj index e8783cc0..428ea60b 100644 --- a/src/yetibot/core/commands/history.clj +++ b/src/yetibot/core/commands/history.clj @@ -149,93 +149,50 @@ ;; build up a vector of query maps using provided `options` we'll ;; merge these into a single map later - extra-queries - (cond-> [{:where/clause - "(is_private = ? OR chat_source_room = ?)" - ;; private history is only available if the commmand - ;; originated from the channel that produced the private - ;; history - :where/args [false (:room chat-source)] - ;; always constrain history to the chat-source that - ;; requested it - :where/map - {:chat_source_adapter (-> chat-source :uuid pr-str)}}] - - ;; and unless the user specified `include-all-channels` or - ;; specific `channels` then we also constrain history to the - ;; channel that it originated from - (and (not (:include-all-channels options)) - (not (:channels options))) (conj - {:where/map - {:chat_source_room - (:room chat-source)}}) - - ;; --channels - (not (blank? (:channels options))) - (conj - (let [cs (split (:channels options) - re-comma-with-maybe-whitespace)] - {:where/clause - (str "(" - (join " OR " - (map (constantly "chat_source_room = ?") cs)) - ")") - :where/args cs})) - - ;; by default we exclude history commands, but this isn't super - ;; cheap, so only exclude history commands if both: - ;; --exclude-commands is not true - since this is much cheaper and - ;; will cover excluding history anyway - ;; --include-history-commands isn't true - (and - (not (:exclude-commands options)) - (not (:include-history-commands - options))) (conj - {:where/clause - "(is_command = ? OR body NOT LIKE ?)" - :where/args [false - (str config-prefix "history%")]}) - - (:exclude-yetibot options) (conj - {:where/clause "is_yetibot = ?" - :where/args [false]}) - - (:exclude-commands options) (conj - {:where/map {:is_command false}}) - - (:exclude-non-commands options) (conj - {:where/map {:is_command true}}) - - ;; --user USER1,USER2 - (not (blank? (:user options))) - (conj - (let [users (split (:user options) - re-comma-with-maybe-whitespace)] - {:where/clause - (str "(" - (join " OR " (map (constantly "user_name = ?") users)) - ")") - :where/args users})) - - ;; --since DATE - (not (blank? (:since options))) - (conj - {:where/clause - "created_at AT TIME ZONE 'UTC' >= ?::TIMESTAMP WITH TIME ZONE" - :where/args [(:since options)]}) - - ;; --until DATE - (not (blank? (:until options))) - (conj - {:where/clause - "created_at AT TIME ZONE 'UTC' <= ?::TIMESTAMP WITH TIME ZONE" - :where/args [(:until options)]})) - - ;; now merge the vector of maps into one single map - - _ (info "extra queries to merge" (pr-str extra-queries)) - extra-query (apply merge-queries extra-queries) - _ (info "extra query" (pr-str extra-query)) + + extra-query {:where/clause + "(is_private = ? OR chat_source_room = ?)" + ;; private history is only available if the commmand + ;; originated from the channel that produced the private + ;; history + :where/args [false (:room chat-source)] + ;; always constrain history to the chat-source that + ;; requested it + :where/map + (merge {:chat_source_adapter + (-> chat-source :uuid pr-str)} + + ;; unless the user specified + ;; `include-all-channels` or specific `channels` + ;; then we also constrain history to the channel + ;; that it originated from + (when (and (not (:include-all-channels options)) + (not (:channels options))) + {:chat_source_room (:room chat-source)}))} + + history-query (h/build-query + {:include-history-commands? + (:include-history-commands options) + :exclude-yetibot? (:exclude-yetibot options) + :exclude-commands? (:exclude-commands options) + :exclude-non-commands? (:exclude-non-commands + options) + :search-query nil + :adapters-filter nil + :channels-filter (when-not (blank? (:channels + options)) + (split + (:channels options) + re-comma-with-maybe-whitespace)) + :users-filter (when-not (blank? (:user options)) + (split + (:user options) + re-comma-with-maybe-whitespace)) + :since-datetime (:since options) + :until-datetime (:until options) + :extra-query extra-query}) + + _ (info "history query" (pr-str history-query)) next-commands (take 1 next-cmds) @@ -243,10 +200,10 @@ (do (reset! skip-next-n skip-n) (history-for-cmd-sequence next-commands - extra-query)) + history-query)) ;; default to last 30 items if there were no filters (take - 30 (query extra-query)))] + 30 (query history-query)))] (debug "computed history" (pr-str history)) ;; format diff --git a/src/yetibot/core/db/util.clj b/src/yetibot/core/db/util.clj index 89171adf..3de5472f 100644 --- a/src/yetibot/core/db/util.clj +++ b/src/yetibot/core/db/util.clj @@ -169,3 +169,9 @@ (-> (query table {:select/clause "COUNT(*) as count"}) first :count)) + +(defn where-eq-any + [column values] + {:where/clause + (str "(" (join " OR " (map (constantly (str column " = ?")) values)) ")") + :where/args values}) diff --git a/src/yetibot/core/models/history.clj b/src/yetibot/core/models/history.clj index 16be12ee..326a5539 100644 --- a/src/yetibot/core/models/history.clj +++ b/src/yetibot/core/models/history.clj @@ -1,22 +1,96 @@ (ns yetibot.core.models.history (:require - [yetibot.core.util.command :refer [extract-command]] - [yetibot.core.db.util :refer [transform-where-map merge-queries]] - [yetibot.core.db.history :refer [create query]] - [clojure.string :refer [join split]] - [yetibot.core.util.time :as t] - [clj-time - [coerce :refer [from-date to-sql-time]] - [format :refer [formatter unparse]] - [core :refer [day year month - to-time-zone after? - default-time-zone now time-zone-for-id date-time utc - ago hours days weeks years months]]] - [yetibot.core.models.users :as u] - [taoensso.timbre :refer [info color-str warn error spy]])) + [yetibot.core.util.command :refer [config-prefix extract-command]] + [yetibot.core.db.util :refer [merge-queries + where-eq-any + transform-where-map merge-queries]] + [yetibot.core.db.history :refer [create query]] + [clojure.string :refer [join split]] + [yetibot.core.util.time :as t] + [clj-time + [coerce :refer [from-date to-sql-time]] + [format :refer [formatter unparse]] + [core :refer [day year month + to-time-zone after? + default-time-zone now time-zone-for-id date-time utc + ago hours days weeks years months]]] + [yetibot.core.models.users :as u] + [taoensso.timbre :refer [info color-str warn error spy]])) ;;;; read +(defn build-query + "Given a bunch of options specific to history return a query data structure" + [{:keys [cursor + exclude-private? + ;; history commands are not included by default + include-history-commands? + exclude-yetibot? + exclude-commands? + exclude-non-commands? + search-query + ;; collection of adapter strings to filter on + adapters-filter + ;; collection of channel strings to filter on + channels-filter + ;; collection of user name strings to filter on + users-filter + ;; datetime + since-datetime + until-datetime + ;; allow caller to pass in additional queries to merge + extra-query] + :as options}] + (info "build query with" (pr-str options)) + (let [queries-to-merge + ;; Start with an empty vector and conditionally conj a bunch of query + ;; maps onto it depending on provided options. This vector will then be + ;; merged into a single combined query map. + (cond-> [] + exclude-private? (conj {:where/map {:is_private false}}) + + ;; by default we exclude history commands, but this isn't super + ;; cheap, so only exclude history commands if both: + ;; - exclude-commands? is not true - since this is much cheaper and + ;; will cover excluding history anyway + ;; - include-history-commands? isn't true + (and + (not exclude-commands?) + (not include-history-commands?)) + (conj + {:where/clause + "(is_command = ? OR body NOT LIKE ?)" + :where/args [false + (str config-prefix "history%")]}) + + exclude-yetibot? (conj {:where/map {:is_yetibot false}}) + exclude-commands? (conj {:where/map {:is_command false}}) + exclude-non-commands? (conj {:where/map {:is_command true}}) + + search-query (conj {:where/clause "body ~ ?" + :where/args [search-query]}) + + cursor "TODO" + adapters-filter (conj (where-eq-any "chat_source_adapter" + adapters-filter)) + channels-filter (conj (where-eq-any "chat_source_room" + channels-filter)) + users-filter (conj (where-eq-any "user_name" users-filter)) + ;; datetime + since-datetime + (conj + {:where/clause + "created_at AT TIME ZONE 'UTC' >= ?::TIMESTAMP WITH TIME ZONE" + :where/args [since-datetime]}) + + until-datetime + (conj + {:where/clause + "created_at AT TIME ZONE 'UTC' <= ?::TIMESTAMP WITH TIME ZONE" + :where/args [(:until options)]}))] + (info "queries-to-merge" (pr-str queries-to-merge)) + (apply merge-queries (conj queries-to-merge extra-query)))) + (defn flatten-one [n] (if (= 1 n) first identity)) (defn count-entities diff --git a/src/yetibot/core/webapp/resolvers/history.clj b/src/yetibot/core/webapp/resolvers/history.clj index e7869c72..e324bd87 100644 --- a/src/yetibot/core/webapp/resolvers/history.clj +++ b/src/yetibot/core/webapp/resolvers/history.clj @@ -1,36 +1,57 @@ (ns yetibot.core.webapp.resolvers.history (:require - [yetibot.core.db.history :refer [query]] - [yetibot.core.models.history :as history] + [com.walmartlabs.lacinia.executor :refer [selections-seq selects-field?]] + [taoensso.timbre :refer [error debug info color-str]] + [yetibot.core.adapters.adapter :as adapter] + [yetibot.core.commands.uptime :as uptime] [yetibot.core.db.alias :as db.alias] - [yetibot.core.db.observe :as db.observe] [yetibot.core.db.cron :as db.cron] - [yetibot.core.commands.uptime :as uptime] - [yetibot.core.adapters.adapter :as adapter] - [yetibot.core.models.users :as users] - [com.walmartlabs.lacinia.executor :refer [selections-seq selects-field?]] - [taoensso.timbre :refer [error debug info color-str]])) + [yetibot.core.db.history :refer [query]] + [yetibot.core.db.observe :as db.observe] + [yetibot.core.models.history :as history] + [yetibot.core.models.users :as users])) (defn history-resolver [context - {:keys [offset limit chat_source_room chat_source_adapter commands_only - yetibot_only search_query user_filter channel_filter] + {limit :first + :keys [cursor + + adapters_filter + channels_filter + + include_history_commands + exclude_yetibot + exclude_commands + exclude_non_commands + + search_query + users_filter + + since_datetime + until_datetime] :as args} value] (info "history resolver. args" args) - (let [where-map (merge {"is_private" false} - (when commands_only {"is_command" true}) - (when yetibot_only {"is_yetibot" true})) - where-clause (when search_query - {:where/clause - "to_tsvector(body) @@ plainto_tsquery(?)" - :where/args [search_query]})] - (query (merge {:query/identifiers identity - :where/map where-map - :limit/clause limit - :offset/clause offset - :order/clause "created_at DESC"} - where-clause)))) + (let [extra-query {:order/clause "created_at DESC" + :limit/clause (or limit 50)} + history-query (history/build-query + {:extra-query extra-query + :cursor cursor + :exclude-private? true + :include-history-commands? include_history_commands + :exclude-yetibot? exclude_yetibot + :exclude-commands? exclude_commands + :exclude-non-commands? exclude_non_commands + :search-query search_query + :adapters-filter adapters_filter + :channels-filter channels_filter + :users-filter users_filter + :since-datetime since_datetime + :until-datetime until_datetime})] + {:history + (query (merge {:query/identifiers identity} + history-query))})) + (defn history-item-resolver [_ {:keys [id] :as args} _] diff --git a/test/yetibot/core/test/commands/history.clj b/test/yetibot/core/test/commands/history.clj index fbbbcc98..3e664f24 100644 --- a/test/yetibot/core/test/commands/history.clj +++ b/test/yetibot/core/test/commands/history.clj @@ -29,9 +29,10 @@ :skip-next-n (atom 0)}) => "336" (provided (query - {:where/clause "(is_private = ? OR chat_source_room = ?) AND (is_command = ? OR body NOT LIKE ?) AND is_yetibot = ?" - :where/args [false "foo" false "!history%" false] - :where/map {:chat_source_adapter ":test" + {:where/clause "(is_command = ? OR body NOT LIKE ?) AND (is_private = ? OR chat_source_room = ?)" + :where/args [false "!history%" false "foo"] + :where/map {:is_yetibot false + :chat_source_adapter ":test" :chat_source_room "foo"} :select/clause "COUNT(*) as count"}) => '({:count 336}))) @@ -70,9 +71,10 @@ :skip-next-n (atom 0)}) => (value "test in foo at 05:03 PM 05/13: !poke") (provided (query - {:where/clause "(is_private = ? OR chat_source_room = ?) AND (is_command = ? OR body NOT LIKE ?) AND is_yetibot = ?" - :where/args [false "foo" false "!history%" false] - :where/map {:chat_source_adapter ":test" + {:where/clause "(is_command = ? OR body NOT LIKE ?) AND (is_private = ? OR chat_source_room = ?)" + :where/args [false "!history%" false "foo"] + :where/map {:is_yetibot false + :chat_source_adapter ":test" :chat_source_room "foo"} :limit/clause "1" :order/clause "created_at DESC"}) => @@ -129,11 +131,11 @@ :skip-next-n (atom 0)}) => (value "test in foo at 02:16 PM 12/04: !echo") (provided (query - {:where/clause "(is_private = ? OR chat_source_room = ?) AND (is_command = ? OR body NOT LIKE ?)" - :where/args [false "foo" false "!history%"] - :where/map {:chat_source_adapter ":test" - :chat_source_room "foo" - :is_command true} + {:where/clause "(is_command = ? OR body NOT LIKE ?) AND (is_private = ? OR chat_source_room = ?)" + :where/args [false "!history%" false "foo"] + :where/map {:is_command true + :chat_source_adapter ":test" + :chat_source_room "foo"} :limit/clause "1" :order/clause "random()"}) => '({:is-command true, @@ -162,8 +164,8 @@ "devth in local at 03:09 PM 03/19: !foo | echo bar %s baz")) (provided (query - #:where{:clause "(is_private = ? OR chat_source_room = ?) AND (is_command = ? OR body NOT LIKE ?) AND body ~ ?" - :args [false "foo" false "!history%" "foo"] + #:where{:clause "(is_command = ? OR body NOT LIKE ?) AND (is_private = ? OR chat_source_room = ?) AND body ~ ?" + :args [false "!history%" false "foo" "foo"] :map {:chat_source_adapter ":test"}}) => '({:is-command true, :is-private true, @@ -217,8 +219,8 @@ (value "devth in local at 05:14 PM 02/19: !channel settings") (provided (query - {:where/clause "(is_private = ? OR chat_source_room = ?) AND (chat_source_room = ?) AND (is_command = ? OR body NOT LIKE ?)" - :where/args [false "foo" "local" false "!history%"] + {:where/clause "(is_command = ? OR body NOT LIKE ?) AND (chat_source_room = ?) AND (is_private = ? OR chat_source_room = ?)" + :where/args [false "!history%" "local" false "foo"] :where/map {:chat_source_adapter ":test"} :limit/clause "1"}) => '({:is-command true, @@ -245,8 +247,8 @@ (value "devth in #obs at 03:50 PM 12/04: !status hi") (provided (query - {:where/clause "(is_private = ? OR chat_source_room = ?) AND (user_name = ? OR user_name = ?)" - :where/args [false "foo" "devth" "yetibot"] + {:where/clause "(user_name = ? OR user_name = ?) AND (is_private = ? OR chat_source_room = ?)" + :where/args ["devth" "yetibot" false "foo"] :where/map {:chat_source_adapter ":test"} :limit/clause "1"}) => '({:is-command true, @@ -273,8 +275,8 @@ (value "yetibot-devth in local at 05:42 PM 05/16: pd teams # list PagerDuty teams\npd teams # list PagerDuty teams matching \npd users # list PagerDuty users\npd users # list PagerDuty users matching ") (provided (query - {:where/clause "(is_private = ? OR chat_source_room = ?) AND created_at AT TIME ZONE 'UTC' >= ?::TIMESTAMP WITH TIME ZONE" - :where/args [false "foo" "2019-03-20T00:00:00.000Z"] + {:where/clause "created_at AT TIME ZONE 'UTC' >= ?::TIMESTAMP WITH TIME ZONE AND (is_private = ? OR chat_source_room = ?)" + :where/args ["2019-03-20T00:00:00.000Z" false "foo"] :where/map {:chat_source_adapter ":test"} :limit/clause "1" :order/clause "created_at DESC"}) => @@ -303,8 +305,8 @@ (value "yetibot-dev in local at 04:28 PM 03/19: Yellowstone, MT (US)\n39.0°F - Clear Sky\nFeels like 35°F\nWinds 1.4 mph WSW") (provided (query - {:where/clause "(is_private = ? OR chat_source_room = ?) AND created_at AT TIME ZONE 'UTC' <= ?::TIMESTAMP WITH TIME ZONE" - :where/args [false "foo" "2019-03-20T00:00:00.000Z"] + {:where/clause "created_at AT TIME ZONE 'UTC' <= ?::TIMESTAMP WITH TIME ZONE AND (is_private = ? OR chat_source_room = ?)" + :where/args [nil false "foo"] :where/map {:chat_source_adapter ":test"} :limit/clause "1" :order/clause "created_at DESC"}) => diff --git a/test/yetibot/core/test/db/util.clj b/test/yetibot/core/test/db/util.clj index 029fb145..f08097cf 100644 --- a/test/yetibot/core/test/db/util.clj +++ b/test/yetibot/core/test/db/util.clj @@ -48,3 +48,8 @@ :where/clause "command LIKE ? AND created_at > ? AND is_yetibot IS ?", :where/args ["likethis" "yesterday" "false"], :select/clause "foo, bar, baz"})))) + +(deftest where-eq-any-test + (is + (= (db.util/where-eq-any "foo" [1 2 3]) + #:where{:clause "(foo = ? OR foo = ? OR foo = ?)", :args [1 2 3]}))) diff --git a/test/yetibot/core/test/webapp/routes/graphql.clj b/test/yetibot/core/test/webapp/routes/graphql.clj index 603992bb..08c21b6f 100644 --- a/test/yetibot/core/test/webapp/routes/graphql.clj +++ b/test/yetibot/core/test/webapp/routes/graphql.clj @@ -29,22 +29,25 @@ (fact "graphql can filter history" (graphql " - history(limit: 30, - offset: 0, - commands_only: $commands_only, - yetibot_only: $yetibot_only, - search_query: $search_query - ) { - id - chat_source_adapter - chat_source_room - command - correlation_id - created_at - user_name - is_command - is_yetibot - body - user_id - user_name + query { + history(first: 50, exclude_yetibot: true) { + page_info { + total_results + } + history { + id + chat_source_adapter + chat_source_room + command + correlation_id + created_at + user_name + is_command + is_yetibot + body + user_id + user_name + } + } }") + ) From 9f22d794655f5c27178cb253c4bf3eab6b73d45b Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Fri, 14 Jun 2019 12:05:09 -0600 Subject: [PATCH 05/10] Implement cursor on history model build-query --- project.clj | 1 + src/yetibot/core/models/history.clj | 18 ++++++- test/yetibot/core/test/commands/history.clj | 3 +- test/yetibot/core/test/models/history.clj | 59 +++++++++------------ 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/project.clj b/project.clj index b5acd215..505ba1f6 100644 --- a/project.clj +++ b/project.clj @@ -130,6 +130,7 @@ [json-path "1.0.1"] ; utils + [org.clojure/data.codec "0.1.1"] ; base64 [funcool/cuerdas "2.1.0"] [clj-stacktrace "0.2.8"] [clj-fuzzy "0.4.1"] diff --git a/src/yetibot/core/models/history.clj b/src/yetibot/core/models/history.clj index 326a5539..fead16f7 100644 --- a/src/yetibot/core/models/history.clj +++ b/src/yetibot/core/models/history.clj @@ -14,11 +14,22 @@ to-time-zone after? default-time-zone now time-zone-for-id date-time utc ago hours days weeks years months]]] + [clojure.data.codec.base64 :as b64] [yetibot.core.models.users :as u] [taoensso.timbre :refer [info color-str warn error spy]])) ;;;; read +(defn cursor->id [cursor] + (try + (read-string (String. (b64/decode (.getBytes cursor)))) + (catch Exception e + (throw (ex-info "Invalid cursor" + {:cursor cursor}))))) + +(defn id->cursor [id] + (String. (b64/encode (.getBytes (str id))))) + (defn build-query "Given a bunch of options specific to history return a query data structure" [{:keys [cursor @@ -42,7 +53,8 @@ extra-query] :as options}] (info "build query with" (pr-str options)) - (let [queries-to-merge + (let [id-from-cursor (when cursor (cursor->id cursor)) + queries-to-merge ;; Start with an empty vector and conditionally conj a bunch of query ;; maps onto it depending on provided options. This vector will then be ;; merged into a single combined query map. @@ -70,7 +82,9 @@ search-query (conj {:where/clause "body ~ ?" :where/args [search-query]}) - cursor "TODO" + id-from-cursor (conj {:where/clause "id >= ?" + :where/args [id-from-cursor]}) + adapters-filter (conj (where-eq-any "chat_source_adapter" adapters-filter)) channels-filter (conj (where-eq-any "chat_source_room" diff --git a/test/yetibot/core/test/commands/history.clj b/test/yetibot/core/test/commands/history.clj index 3e664f24..378fd991 100644 --- a/test/yetibot/core/test/commands/history.clj +++ b/test/yetibot/core/test/commands/history.clj @@ -2,8 +2,7 @@ (:require [yetibot.core.db :as db] [yetibot.core.midje :refer [value data]] - [midje.sweet :refer [namespace-state-changes with-state-changes fact => - facts truthy]] + [midje.sweet :refer [fact => facts]] [yetibot.core.models.history :as h] [yetibot.core.db.history :refer [query]] [yetibot.core.commands.history :refer :all])) diff --git a/test/yetibot/core/test/models/history.clj b/test/yetibot/core/test/models/history.clj index 14709e96..21ab7efb 100644 --- a/test/yetibot/core/test/models/history.clj +++ b/test/yetibot/core/test/models/history.clj @@ -1,10 +1,13 @@ (ns yetibot.core.test.models.history (:require - [clojure.java.jdbc :as sql] - [yetibot.core.models.history :refer :all] - [yetibot.core.db :as db] - [yetibot.core.util :refer [is-command?]] - [clojure.test :refer :all])) + [clojure.java.jdbc :as sql] + [yetibot.core.models.history :refer :all] + [yetibot.core.db :as db] + [yetibot.core.util :refer [is-command?]] + [yetibot.core.midje :refer [value data]] + [clojure.test.check.generators :as gen] + [midje.experimental :refer [for-all]] + [midje.sweet :refer [fact => facts]])) (def chat-source {:adapter :slack :uuid :test :room "foo"}) @@ -19,7 +22,7 @@ ;; investigate embedded postgres as a solution: ;; https://eli.naeher.name/embedded-postgres-in-clojure/ ;; we need a database -(db/start) +(defonce db-start (db/start)) ;; Another quick option would be to use a fixture to populate then rollback ;; after the tests, something like: @@ -39,34 +42,24 @@ (use-fixtures :once populate) -(def extra-query {:where/map {:chat-source-adapter (-> chat-source :uuid pr-str) - :is-yetibot false}}) +(def extra-query + {:where/map {:chat-source-adapter (-> chat-source :uuid pr-str) + :is-yetibot false}}) -(deftest test-count-entities - (count-entities extra-query)) +(for-all [positive-num gen/s-pos-int] + (fact "cursor to id transformations are isomorphic" + (-> positive-num id->cursor cursor->id) => positive-num)) -(deftest test-head - (head 2 extra-query)) +(fact "build query works with cursors" + (build-query {:cursor (id->cursor 10)})) -(deftest test-tail - (count (tail 2 extra-query))) - -(deftest test-random - (map? (random extra-query))) - -(deftest test-grep - (grep "b.d+" extra-query)) - -(deftest test-cmd-only-items - (cmd-only-items chat-source)) - -(deftest test-non-cmd-items - (non-cmd-items chat-source)) - -(deftest history-should-be-in-order - ;; TODO - ) - -(deftest last-chat-for-channel-test +(comment + ;; scratch - these could be ported to tests + (head 2 extra-query) + (count (tail 2 extra-query)) + (grep "b.d+" extra-query) + (cmd-only-items chat-source) + (non-cmd-items chat-source) (last-chat-for-channel chat-source true) - (last-chat-for-channel chat-source false)) + (last-chat-for-channel chat-source false) + (map? (random extra-query))) From 33f808d7239624b76400eccc7f00b1aa48b1c7ac Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Mon, 17 Jun 2019 13:39:32 -0600 Subject: [PATCH 06/10] Start adding page info on history resolver --- src/yetibot/core/webapp/resolvers/history.clj | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/yetibot/core/webapp/resolvers/history.clj b/src/yetibot/core/webapp/resolvers/history.clj index e324bd87..87ded1c5 100644 --- a/src/yetibot/core/webapp/resolvers/history.clj +++ b/src/yetibot/core/webapp/resolvers/history.clj @@ -50,7 +50,13 @@ :until-datetime until_datetime})] {:history (query (merge {:query/identifiers identity} - history-query))})) + history-query)) + ;; TODO lazily retrieve these + :page_info {:total_results (history/count-entities history-query) + :next_page_cursor "TODO" + :has_next_page "TODO" + } + })) (defn history-item-resolver From ea56c320cdc2afb5b407e15e9961828b7013324f Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Mon, 24 Jun 2019 16:07:31 -0600 Subject: [PATCH 07/10] Implement page_info on paged_history --- src/yetibot/core/chat.clj | 3 +- src/yetibot/core/models/history.clj | 22 +++++- src/yetibot/core/webapp/resolvers/history.clj | 76 ++++++++++++------- test/yetibot/core/test/models/history.clj | 2 +- 4 files changed, 73 insertions(+), 30 deletions(-) diff --git a/src/yetibot/core/chat.clj b/src/yetibot/core/chat.clj index e76ba2b7..c1b682ae 100644 --- a/src/yetibot/core/chat.clj +++ b/src/yetibot/core/chat.clj @@ -56,7 +56,8 @@ :uuid (a/uuid *adapter*)}) (defn chat-source - "Data structure representing the chat adapter and channel that a message came from" + "Data structure representing the chat adapter and channel that a message came + from" [channel] (merge (base-chat-source) {:room channel})) ; (defn set-room-broadcast [room broadcast?] ((:set-room-broadcast *messaging-fns*) room broadcast?)) diff --git a/src/yetibot/core/models/history.clj b/src/yetibot/core/models/history.clj index fead16f7..cd8577ae 100644 --- a/src/yetibot/core/models/history.clj +++ b/src/yetibot/core/models/history.clj @@ -82,7 +82,17 @@ search-query (conj {:where/clause "body ~ ?" :where/args [search-query]}) - id-from-cursor (conj {:where/clause "id >= ?" + ;; >= for ASC + ;; <= for DESC + id-from-cursor (conj {:where/clause + (str + "id " + (if (re-find + #"DESC" + (or (:order/clause extra-query) "")) + "<=" ;; DESC + ">=") ;; ASC + " ?") :where/args [id-from-cursor]}) adapters-filter (conj (where-eq-any "chat_source_adapter" @@ -253,3 +263,13 @@ (create (assoc history-item :chat-source-adapter (pr-str chat-source-adapter)))) + + +(comment + (count-entities {}) + (count-entities + {:where/map {:is_private false} + :where/clause "(is_command = ? OR body NOT LIKE ?)" + :where/args [false "!history%"] + :limit/clause 51 + :order/clause "created_at DESC"})) diff --git a/src/yetibot/core/webapp/resolvers/history.clj b/src/yetibot/core/webapp/resolvers/history.clj index 87ded1c5..483f1820 100644 --- a/src/yetibot/core/webapp/resolvers/history.clj +++ b/src/yetibot/core/webapp/resolvers/history.clj @@ -1,6 +1,8 @@ (ns yetibot.core.webapp.resolvers.history (:require - [com.walmartlabs.lacinia.executor :refer [selections-seq selects-field?]] + [com.walmartlabs.lacinia.executor :refer [selections-tree + selections-seq + selects-field?]] [taoensso.timbre :refer [error debug info color-str]] [yetibot.core.adapters.adapter :as adapter] [yetibot.core.commands.uptime :as uptime] @@ -31,32 +33,52 @@ until_datetime] :as args} value] - (info "history resolver. args" args) - (let [extra-query {:order/clause "created_at DESC" - :limit/clause (or limit 50)} - history-query (history/build-query - {:extra-query extra-query - :cursor cursor - :exclude-private? true - :include-history-commands? include_history_commands - :exclude-yetibot? exclude_yetibot - :exclude-commands? exclude_commands - :exclude-non-commands? exclude_non_commands - :search-query search_query - :adapters-filter adapters_filter - :channels-filter channels_filter - :users-filter users_filter - :since-datetime since_datetime - :until-datetime until_datetime})] - {:history - (query (merge {:query/identifiers identity} - history-query)) - ;; TODO lazily retrieve these - :page_info {:total_results (history/count-entities history-query) - :next_page_cursor "TODO" - :has_next_page "TODO" - } - })) + (info "history resolver" + {:args args + :selections-seq (selections-seq context) + :selections-tree (selections-tree context)}) + (let [limit (or limit 50) + extra-query {:order/clause "created_at DESC" + ;; fetch one extra record so we can compute the next hash + :limit/clause (inc limit)} + + ;; history query without extra-query or cursor for counting + base-query-options {:exclude-private? true + :include-history-commands? include_history_commands + :exclude-yetibot? exclude_yetibot + :exclude-commands? exclude_commands + :exclude-non-commands? exclude_non_commands + :search-query search_query + :adapters-filter adapters_filter + :channels-filter channels_filter + :users-filter users_filter + :since-datetime since_datetime + :until-datetime until_datetime} + + history-query (history/build-query (merge base-query-options + {:extra-query extra-query + :cursor cursor})) + + results (query (merge {:query/identifiers identity} + history-query)) + has-next-page? (> (count results) limit) + next-page-cursor (when has-next-page? + (-> results + last + :id + history/id->cursor))] + + (info "history-query" (pr-str history-query)) + + {;; only take up to the limit since there may be an extra record in the + ;; result set + :history (take limit results) + :page_info {:total_results (when (selects-field? + context :page_info/total_results) + (history/count-entities + (history/build-query base-query-options))) + :next_page_cursor next-page-cursor + :has_next_page has-next-page?}})) (defn history-item-resolver diff --git a/test/yetibot/core/test/models/history.clj b/test/yetibot/core/test/models/history.clj index 21ab7efb..d467b3b2 100644 --- a/test/yetibot/core/test/models/history.clj +++ b/test/yetibot/core/test/models/history.clj @@ -40,7 +40,7 @@ ;; (run! add-history ["!echo" "!status" "!poke"]) (f)) -(use-fixtures :once populate) +;; (use-fixtures :once populate) (def extra-query {:where/map {:chat-source-adapter (-> chat-source :uuid pr-str) From d7d5822a5482988f3fd8004b621f1f6b40896d7c Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Mon, 24 Jun 2019 16:09:21 -0600 Subject: [PATCH 08/10] Cleanup --- src/yetibot/core/models/history.clj | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/yetibot/core/models/history.clj b/src/yetibot/core/models/history.clj index cd8577ae..a38e47bf 100644 --- a/src/yetibot/core/models/history.clj +++ b/src/yetibot/core/models/history.clj @@ -263,13 +263,3 @@ (create (assoc history-item :chat-source-adapter (pr-str chat-source-adapter)))) - - -(comment - (count-entities {}) - (count-entities - {:where/map {:is_private false} - :where/clause "(is_command = ? OR body NOT LIKE ?)" - :where/args [false "!history%"] - :limit/clause 51 - :order/clause "created_at DESC"})) From 609456683c1cefb54a9ea660a968983d00f1279c Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Mon, 24 Jun 2019 16:11:07 -0600 Subject: [PATCH 09/10] Cleanup schema --- resources/graphql-schema.edn | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/resources/graphql-schema.edn b/resources/graphql-schema.edn index ffa4e542..9c77318c 100644 --- a/resources/graphql-schema.edn +++ b/resources/graphql-schema.edn @@ -23,25 +23,6 @@ {:page_info {:type :page_info} :history {:type (list :history)}}} - ;; { - ;; hero { - ;; name - ;; friendsConnection(first:2 after:"Y3Vyc29yMQ==") { - ;; totalCount - ;; edges { - ;; node { - ;; name - ;; } - ;; cursor - ;; } - ;; pageInfo { - ;; endCursor - ;; hasNextPage - ;; } - ;; } - ;; } - ;; } - :stats {:fields {:uptime {:type String} :adapter_count {:type Int} From 2d197da584cf3e97343d8f32b48922bb77ba2b86 Mon Sep 17 00:00:00 2001 From: Trevor Hartman Date: Tue, 25 Jun 2019 11:55:42 -0600 Subject: [PATCH 10/10] Bring adapters gql resource back --- resources/graphql-schema.edn | 19 +++++++++++--- src/yetibot/core/webapp/resolvers/history.clj | 3 ++- src/yetibot/core/webapp/routes/graphql.clj | 26 ++++++++++--------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/resources/graphql-schema.edn b/resources/graphql-schema.edn index 9c77318c..921bb6af 100644 --- a/resources/graphql-schema.edn +++ b/resources/graphql-schema.edn @@ -23,6 +23,12 @@ {:page_info {:type :page_info} :history {:type (list :history)}}} + :adapter {:fields {:uuid {:type String} + :platform {:type String} + :is_connected {:type Boolean} + :connection_latency {:type Int} + :connection_last_active_timestamp {:type String}}} + :stats {:fields {:uptime {:type String} :adapter_count {:type Int} @@ -69,10 +75,9 @@ :schedule {:type String} :cmd {:type String}}} - ;; TODO add adapter info back - :adapter {:fields - {:chat_source_adapter {:type String} - :channels {:type (list :channel)}}} + :adapter_channel {:fields + {:chat_source_adapter {:type String} + :channels {:type (list :channel)}}} :channel {:fields {:chat_source_adapter {:type String} @@ -92,9 +97,15 @@ :args {:expr {:type String}} :resolve :eval} + ;; info about the configured adapters and their connection status + ;; :adapters :adapters {:type (list :adapter) :resolve :adapters} + ;; the list of channels by adapter + :adapter_channels {:type (list :adapter_channel) + :resolve :adapter_channels} + :history {:type :paged_history :resolve :history :args {;; pagination diff --git a/src/yetibot/core/webapp/resolvers/history.clj b/src/yetibot/core/webapp/resolvers/history.clj index 483f1820..00d1a457 100644 --- a/src/yetibot/core/webapp/resolvers/history.clj +++ b/src/yetibot/core/webapp/resolvers/history.clj @@ -88,7 +88,8 @@ (query (merge {:query/identifiers identity :where/map where-map}))))) -(defn adapters-resolver +(defn adapter-channels-resolver + "List the channels by grouped by adapter" [_ _ _] (for [[k v] (group-by :chat_source_adapter diff --git a/src/yetibot/core/webapp/routes/graphql.clj b/src/yetibot/core/webapp/routes/graphql.clj index d9d974d5..a1ff89ef 100644 --- a/src/yetibot/core/webapp/routes/graphql.clj +++ b/src/yetibot/core/webapp/routes/graphql.clj @@ -11,7 +11,7 @@ [taoensso.timbre :refer [error debug info color-str]] [yetibot.core.webapp.resolvers :as resolvers] [yetibot.core.webapp.resolvers.stats :refer [stats-resolver]] - [yetibot.core.webapp.resolvers.history :refer [adapters-resolver + [yetibot.core.webapp.resolvers.history :refer [adapter-channels-resolver history-resolver history-item-resolver]])) @@ -20,17 +20,19 @@ (-> (io/resource "graphql-schema.edn") slurp edn/read-string - (lacinia.util/attach-resolvers {:eval resolvers/eval-resolver - :history history-resolver - :history_item history-item-resolver - :users resolvers/users-resolver - :user resolvers/user-resolver - :stats stats-resolver - :aliases resolvers/aliases-resolver - :observers resolvers/observers-resolver - :crons resolvers/crons-resolver - :adapters adapters-resolver - :karmas resolvers/karmas-resolver}) + (lacinia.util/attach-resolvers + {:eval resolvers/eval-resolver + :history history-resolver + :history_item history-item-resolver + :users resolvers/users-resolver + :user resolvers/user-resolver + :stats stats-resolver + :aliases resolvers/aliases-resolver + :observers resolvers/observers-resolver + :crons resolvers/crons-resolver + :adapter_channels adapter-channels-resolver + :adapters resolvers/adapters-resolver + :karmas resolvers/karmas-resolver}) lacinia.schema/compile)) ;; note this is not reloadable