From 91b4685043bac6b4dd4d0bf6a22a46b5131e5027 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 1 Nov 2023 08:24:19 +0000 Subject: [PATCH] Deploy website - based on 9d7cb130e75b1b7f128ce6808e6c1831eda0edd9 --- 404.html | 4 ++-- assets/js/{07653d02.bbc01eb3.js => 07653d02.dee0a882.js} | 2 +- assets/js/{12ebdcc3.f5034dba.js => 12ebdcc3.d97869cf.js} | 2 +- assets/js/{174ad3e8.2cd20389.js => 174ad3e8.e7d3cc5f.js} | 2 +- assets/js/{239f3910.29ef26a9.js => 239f3910.f3153671.js} | 2 +- assets/js/{260cf76e.d9d2c58f.js => 260cf76e.a8e67b57.js} | 2 +- assets/js/{2be45fc7.dff5e95a.js => 2be45fc7.1fc01053.js} | 2 +- assets/js/{3278defe.4c2877e3.js => 3278defe.98e4e90e.js} | 2 +- assets/js/{420f7ef9.825be566.js => 420f7ef9.16f54e78.js} | 2 +- assets/js/{4f8cfcf0.0e6f5ff6.js => 4f8cfcf0.40f2e70e.js} | 2 +- assets/js/{537b8c88.2571e53c.js => 537b8c88.e93484a0.js} | 2 +- assets/js/{543eea9a.15be6dd8.js => 543eea9a.b6af3155.js} | 2 +- assets/js/{54f44165.30190704.js => 54f44165.0f14f797.js} | 2 +- assets/js/{553314f8.d58c9075.js => 553314f8.b1d4357b.js} | 2 +- assets/js/{588bd741.eb9469ea.js => 588bd741.ed5a26d8.js} | 2 +- assets/js/{5ec01bbe.75441a54.js => 5ec01bbe.f7aa917b.js} | 2 +- assets/js/{65608a03.ac421215.js => 65608a03.b6ae1562.js} | 2 +- assets/js/{6ae6f4b6.23b2ede1.js => 6ae6f4b6.15eed989.js} | 2 +- assets/js/{6e97b392.d9ae6e69.js => 6e97b392.e912cd31.js} | 2 +- assets/js/{79c9e2d7.b6864922.js => 79c9e2d7.4b681d2a.js} | 2 +- assets/js/{7c550269.e31700b4.js => 7c550269.3e410d26.js} | 2 +- assets/js/{7fa9dab1.abec0c1c.js => 7fa9dab1.11b74139.js} | 2 +- assets/js/{81694fd1.0202bb1c.js => 81694fd1.0fa2d851.js} | 2 +- assets/js/{85be924b.c47e1e8b.js => 85be924b.3bd1f4f0.js} | 2 +- assets/js/{8a1416ba.a6a96efb.js => 8a1416ba.eb20bc0d.js} | 2 +- assets/js/{8ce98423.33350de4.js => 8ce98423.3a8598d4.js} | 2 +- assets/js/{9efad6bf.73405ccb.js => 9efad6bf.ba1cc1b9.js} | 2 +- assets/js/{a09c2993.f0ff2f69.js => a09c2993.804b23b7.js} | 2 +- assets/js/{ab079b58.cca0df44.js => ab079b58.658a2062.js} | 2 +- assets/js/{ad92602d.f6d05e6f.js => ad92602d.436c4a20.js} | 2 +- assets/js/{ae53eb13.a144c5b6.js => ae53eb13.d942e424.js} | 2 +- assets/js/{b115b19d.6e9b75a8.js => b115b19d.bcda7022.js} | 2 +- assets/js/{b14ed82f.a4d8cb0e.js => b14ed82f.4df5e09d.js} | 2 +- assets/js/{b35a8f38.0bc983f1.js => b35a8f38.80c646f0.js} | 2 +- assets/js/{b7fe478b.8d28011b.js => b7fe478b.3beababb.js} | 2 +- assets/js/{b95ea484.07276431.js => b95ea484.bfba3a89.js} | 2 +- assets/js/{bf534763.df52fb30.js => bf534763.af003002.js} | 2 +- assets/js/{c20e3951.c0e35d10.js => c20e3951.ace81d80.js} | 2 +- assets/js/{cf34028a.c343a909.js => cf34028a.4ba3eef0.js} | 2 +- assets/js/{d194c8d1.23a61ae6.js => d194c8d1.fef87132.js} | 2 +- assets/js/{d4c33874.7158cdd1.js => d4c33874.1c5e822c.js} | 2 +- assets/js/{d6ec5803.178dc964.js => d6ec5803.0fcb8587.js} | 2 +- assets/js/{e22177c2.9fd87071.js => e22177c2.c87118ee.js} | 2 +- assets/js/{e7bdcf2c.bcf62a64.js => e7bdcf2c.a0590121.js} | 2 +- assets/js/{f00c4084.e225088a.js => f00c4084.397b675f.js} | 2 +- assets/js/{f1cf558f.4ea152c8.js => f1cf558f.77a12ede.js} | 2 +- assets/js/{f2458df1.fa91b064.js => f2458df1.8d5e43d6.js} | 2 +- assets/js/{f25bba18.79e30085.js => f25bba18.3cf9e21b.js} | 2 +- assets/js/{f3b98b79.7222744e.js => f3b98b79.502ed0da.js} | 2 +- assets/js/{fa377e30.b52e703f.js => fa377e30.2fba79ef.js} | 2 +- assets/js/{fc49bffc.25df574d.js => fc49bffc.51ab710e.js} | 2 +- .../{runtime~main.af982d20.js => runtime~main.83747779.js} | 2 +- blog/archive/index.html | 4 ++-- blog/index.html | 4 ++-- blog/introducing-optimus/index.html | 4 ++-- blog/tags/goto/index.html | 4 ++-- blog/tags/index.html | 4 ++-- blog/tags/optimus/index.html | 4 ++-- docs/building-plugin/introduction/index.html | 6 +++--- docs/building-plugin/tutorial/index.html | 6 +++--- docs/client-guide/applying-job-specifications/index.html | 6 +++--- docs/client-guide/backup-bigquery-resource/index.html | 6 +++--- docs/client-guide/configuration/index.html | 6 +++--- docs/client-guide/create-job-specifications/index.html | 6 +++--- docs/client-guide/defining-scheduler-version/index.html | 6 +++--- docs/client-guide/installing-plugin/index.html | 6 +++--- docs/client-guide/manage-bigquery-resource/index.html | 6 +++--- docs/client-guide/managing-project-namespace/index.html | 6 +++--- docs/client-guide/managing-secrets/index.html | 6 +++--- docs/client-guide/organizing-specifications/index.html | 6 +++--- docs/client-guide/replay-a-job/index.html | 6 +++--- docs/client-guide/setting-up-alert/index.html | 6 +++--- docs/client-guide/uploading-jobs-to-scheduler/index.html | 6 +++--- docs/client-guide/verifying-jobs/index.html | 6 +++--- docs/client-guide/work-with-extension/index.html | 6 +++--- docs/concepts/architecture/index.html | 6 +++--- docs/concepts/dependency/index.html | 6 +++--- docs/concepts/intervals-and-windows/index.html | 6 +++--- docs/concepts/job-run/index.html | 6 +++--- docs/concepts/job/index.html | 6 +++--- docs/concepts/macros/index.html | 6 +++--- docs/concepts/namespace/index.html | 6 +++--- docs/concepts/plugin/index.html | 6 +++--- docs/concepts/project/index.html | 6 +++--- docs/concepts/replay-and-backup/index.html | 6 +++--- docs/concepts/resource/index.html | 6 +++--- docs/concepts/secret/index.html | 6 +++--- docs/contribute/contribution-process/index.html | 6 +++--- docs/contribute/developer-env-setup/index.html | 6 +++--- docs/getting-started/installation/index.html | 6 +++--- docs/getting-started/quick-start/index.html | 6 +++--- docs/introduction/index.html | 6 +++--- docs/reference/api/index.html | 6 +++--- docs/reference/faq/index.html | 6 +++--- docs/reference/metrics/index.html | 6 +++--- docs/rfcs/improving_time_and_flow_of_deployment/index.html | 6 +++--- docs/rfcs/index.html | 6 +++--- docs/rfcs/optimus_dashboarding/index.html | 6 +++--- docs/rfcs/replay_rate_limiting/index.html | 6 +++--- .../rfcs/revisit_automated_dependency_resolution/index.html | 6 +++--- docs/rfcs/secret_management/index.html | 6 +++--- docs/rfcs/simplify_plugin_maintenance/index.html | 6 +++--- .../support_for_depending_on_external_sensors/index.html | 6 +++--- docs/rfcs/template/index.html | 6 +++--- docs/server-guide/configuration/index.html | 6 +++--- docs/server-guide/db-migrations/index.html | 6 +++--- docs/server-guide/installing-plugins/index.html | 6 +++--- docs/server-guide/starting-optimus-server/index.html | 6 +++--- help/index.html | 4 ++-- index.html | 4 ++-- 110 files changed, 219 insertions(+), 219 deletions(-) rename assets/js/{07653d02.bbc01eb3.js => 07653d02.dee0a882.js} (98%) rename assets/js/{12ebdcc3.f5034dba.js => 12ebdcc3.d97869cf.js} (99%) rename assets/js/{174ad3e8.2cd20389.js => 174ad3e8.e7d3cc5f.js} (99%) rename assets/js/{239f3910.29ef26a9.js => 239f3910.f3153671.js} (98%) rename assets/js/{260cf76e.d9d2c58f.js => 260cf76e.a8e67b57.js} (98%) rename assets/js/{2be45fc7.dff5e95a.js => 2be45fc7.1fc01053.js} (99%) rename assets/js/{3278defe.4c2877e3.js => 3278defe.98e4e90e.js} (98%) rename assets/js/{420f7ef9.825be566.js => 420f7ef9.16f54e78.js} (98%) rename assets/js/{4f8cfcf0.0e6f5ff6.js => 4f8cfcf0.40f2e70e.js} (98%) rename assets/js/{537b8c88.2571e53c.js => 537b8c88.e93484a0.js} (98%) rename assets/js/{543eea9a.15be6dd8.js => 543eea9a.b6af3155.js} (98%) rename assets/js/{54f44165.30190704.js => 54f44165.0f14f797.js} (98%) rename assets/js/{553314f8.d58c9075.js => 553314f8.b1d4357b.js} (99%) rename assets/js/{588bd741.eb9469ea.js => 588bd741.ed5a26d8.js} (99%) rename assets/js/{5ec01bbe.75441a54.js => 5ec01bbe.f7aa917b.js} (99%) rename assets/js/{65608a03.ac421215.js => 65608a03.b6ae1562.js} (98%) rename assets/js/{6ae6f4b6.23b2ede1.js => 6ae6f4b6.15eed989.js} (98%) rename assets/js/{6e97b392.d9ae6e69.js => 6e97b392.e912cd31.js} (98%) rename assets/js/{79c9e2d7.b6864922.js => 79c9e2d7.4b681d2a.js} (99%) rename assets/js/{7c550269.e31700b4.js => 7c550269.3e410d26.js} (99%) rename assets/js/{7fa9dab1.abec0c1c.js => 7fa9dab1.11b74139.js} (99%) rename assets/js/{81694fd1.0202bb1c.js => 81694fd1.0fa2d851.js} (99%) rename assets/js/{85be924b.c47e1e8b.js => 85be924b.3bd1f4f0.js} (98%) rename assets/js/{8a1416ba.a6a96efb.js => 8a1416ba.eb20bc0d.js} (98%) rename assets/js/{8ce98423.33350de4.js => 8ce98423.3a8598d4.js} (99%) rename assets/js/{9efad6bf.73405ccb.js => 9efad6bf.ba1cc1b9.js} (98%) rename assets/js/{a09c2993.f0ff2f69.js => a09c2993.804b23b7.js} (99%) rename assets/js/{ab079b58.cca0df44.js => ab079b58.658a2062.js} (99%) rename assets/js/{ad92602d.f6d05e6f.js => ad92602d.436c4a20.js} (99%) rename assets/js/{ae53eb13.a144c5b6.js => ae53eb13.d942e424.js} (98%) rename assets/js/{b115b19d.6e9b75a8.js => b115b19d.bcda7022.js} (98%) rename assets/js/{b14ed82f.a4d8cb0e.js => b14ed82f.4df5e09d.js} (98%) rename assets/js/{b35a8f38.0bc983f1.js => b35a8f38.80c646f0.js} (98%) rename assets/js/{b7fe478b.8d28011b.js => b7fe478b.3beababb.js} (98%) rename assets/js/{b95ea484.07276431.js => b95ea484.bfba3a89.js} (98%) rename assets/js/{bf534763.df52fb30.js => bf534763.af003002.js} (99%) rename assets/js/{c20e3951.c0e35d10.js => c20e3951.ace81d80.js} (99%) rename assets/js/{cf34028a.c343a909.js => cf34028a.4ba3eef0.js} (98%) rename assets/js/{d194c8d1.23a61ae6.js => d194c8d1.fef87132.js} (98%) rename assets/js/{d4c33874.7158cdd1.js => d4c33874.1c5e822c.js} (98%) rename assets/js/{d6ec5803.178dc964.js => d6ec5803.0fcb8587.js} (99%) rename assets/js/{e22177c2.9fd87071.js => e22177c2.c87118ee.js} (99%) rename assets/js/{e7bdcf2c.bcf62a64.js => e7bdcf2c.a0590121.js} (98%) rename assets/js/{f00c4084.e225088a.js => f00c4084.397b675f.js} (99%) rename assets/js/{f1cf558f.4ea152c8.js => f1cf558f.77a12ede.js} (98%) rename assets/js/{f2458df1.fa91b064.js => f2458df1.8d5e43d6.js} (98%) rename assets/js/{f25bba18.79e30085.js => f25bba18.3cf9e21b.js} (99%) rename assets/js/{f3b98b79.7222744e.js => f3b98b79.502ed0da.js} (98%) rename assets/js/{fa377e30.b52e703f.js => fa377e30.2fba79ef.js} (98%) rename assets/js/{fc49bffc.25df574d.js => fc49bffc.51ab710e.js} (99%) rename assets/js/{runtime~main.af982d20.js => runtime~main.83747779.js} (81%) diff --git a/404.html b/404.html index efd884d652..b34278abcb 100644 --- a/404.html +++ b/404.html @@ -10,13 +10,13 @@ - +
Skip to main content

Page Not Found

We could not find what you were looking for.

Please contact the owner of the site that linked you to the original URL and let them know their link is broken.

- + \ No newline at end of file diff --git a/assets/js/07653d02.bbc01eb3.js b/assets/js/07653d02.dee0a882.js similarity index 98% rename from assets/js/07653d02.bbc01eb3.js rename to assets/js/07653d02.dee0a882.js index 15d9eebc0e..4d82c3e232 100644 --- a/assets/js/07653d02.bbc01eb3.js +++ b/assets/js/07653d02.dee0a882.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[1033],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>m});var i=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function a(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=i.createContext({}),c=function(e){var t=i.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},u=function(e){var t=c(e.components);return i.createElement(s.Provider,{value:t},e.children)},p="mdxType",g={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},d=i.forwardRef((function(e,t){var n=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),p=c(n),d=r,m=p["".concat(s,".").concat(d)]||p[d]||g[d]||l;return n?i.createElement(m,a(a({ref:t},u),{},{components:n})):i.createElement(m,a({ref:t},u))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=n.length,a=new Array(l);a[0]=d;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o[p]="string"==typeof e?e:r,a[1]=o;for(var c=2;c{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>a,default:()=>g,frontMatter:()=>l,metadata:()=>o,toc:()=>c});var i=n(7462),r=(n(7294),n(3905));const l={},a="Installing Plugin in Client",o={unversionedId:"client-guide/installing-plugin",id:"client-guide/installing-plugin",title:"Installing Plugin in Client",description:"Creating job specifications requires a plugin to be installed in the system caller. To simplify the installation,",source:"@site/docs/client-guide/installing-plugin.md",sourceDirName:"client-guide",slug:"/client-guide/installing-plugin",permalink:"/optimus/docs/client-guide/installing-plugin",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/installing-plugin.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Managing Secrets",permalink:"/optimus/docs/client-guide/managing-secrets"},next:{title:"Manage BigQuery Resource",permalink:"/optimus/docs/client-guide/manage-bigquery-resource"}},s={},c=[],u={toc:c},p="wrapper";function g(e){let{components:t,...n}=e;return(0,r.kt)(p,(0,i.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"installing-plugin-in-client"},"Installing Plugin in Client"),(0,r.kt)("p",null,"Creating job specifications requires a plugin to be installed in the system caller. To simplify the installation,\nOptimus CLI can sync the YAML plugins supported and served by the Optimus server (with host as declared in project\nconfig) using the following command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus plugin sync -c optimus.yaml\n")),(0,r.kt)("p",null,"Note: This will install plugins in the ",(0,r.kt)("inlineCode",{parentName:"p"},".plugins")," folder."))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[1033],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>m});var i=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function a(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=i.createContext({}),c=function(e){var t=i.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},u=function(e){var t=c(e.components);return i.createElement(s.Provider,{value:t},e.children)},p="mdxType",g={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},d=i.forwardRef((function(e,t){var n=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),p=c(n),d=r,m=p["".concat(s,".").concat(d)]||p[d]||g[d]||l;return n?i.createElement(m,a(a({ref:t},u),{},{components:n})):i.createElement(m,a({ref:t},u))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=n.length,a=new Array(l);a[0]=d;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o[p]="string"==typeof e?e:r,a[1]=o;for(var c=2;c{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>a,default:()=>g,frontMatter:()=>l,metadata:()=>o,toc:()=>c});var i=n(7462),r=(n(7294),n(3905));const l={},a="Installing Plugin in Client",o={unversionedId:"client-guide/installing-plugin",id:"client-guide/installing-plugin",title:"Installing Plugin in Client",description:"Creating job specifications requires a plugin to be installed in the system caller. To simplify the installation,",source:"@site/docs/client-guide/installing-plugin.md",sourceDirName:"client-guide",slug:"/client-guide/installing-plugin",permalink:"/optimus/docs/client-guide/installing-plugin",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/installing-plugin.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Managing Secrets",permalink:"/optimus/docs/client-guide/managing-secrets"},next:{title:"Manage BigQuery Resource",permalink:"/optimus/docs/client-guide/manage-bigquery-resource"}},s={},c=[],u={toc:c},p="wrapper";function g(e){let{components:t,...n}=e;return(0,r.kt)(p,(0,i.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"installing-plugin-in-client"},"Installing Plugin in Client"),(0,r.kt)("p",null,"Creating job specifications requires a plugin to be installed in the system caller. To simplify the installation,\nOptimus CLI can sync the YAML plugins supported and served by the Optimus server (with host as declared in project\nconfig) using the following command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus plugin sync -c optimus.yaml\n")),(0,r.kt)("p",null,"Note: This will install plugins in the ",(0,r.kt)("inlineCode",{parentName:"p"},".plugins")," folder."))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/12ebdcc3.f5034dba.js b/assets/js/12ebdcc3.d97869cf.js similarity index 99% rename from assets/js/12ebdcc3.f5034dba.js rename to assets/js/12ebdcc3.d97869cf.js index 328e476778..fa5a0d6968 100644 --- a/assets/js/12ebdcc3.f5034dba.js +++ b/assets/js/12ebdcc3.d97869cf.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2106],{3905:(e,t,n)=>{n.d(t,{Zo:()=>d,kt:()=>m});var a=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=a.createContext({}),u=function(e){var t=a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=u(e.components);return a.createElement(l.Provider,{value:t},e.children)},c="mdxType",p={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},h=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,l=e.parentName,d=s(e,["components","mdxType","originalType","parentName"]),c=u(n),h=o,m=c["".concat(l,".").concat(h)]||c[h]||p[h]||r;return n?a.createElement(m,i(i({ref:t},d),{},{components:n})):a.createElement(m,i({ref:t},d))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=h;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[c]="string"==typeof e?e:o,i[1]=s;for(var u=2;u{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>i,default:()=>p,frontMatter:()=>r,metadata:()=>s,toc:()=>u});var a=n(7462),o=(n(7294),n(3905));const r={},i=void 0,s={unversionedId:"rfcs/revisit_automated_dependency_resolution",id:"rfcs/revisit_automated_dependency_resolution",title:"revisit_automated_dependency_resolution",description:"- Feature Name: Revisit Automated Dependency Resolution Logic",source:"@site/docs/rfcs/20220124_revisit_automated_dependency_resolution.md",sourceDirName:"rfcs",slug:"/rfcs/revisit_automated_dependency_resolution",permalink:"/optimus/docs/rfcs/revisit_automated_dependency_resolution",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220124_revisit_automated_dependency_resolution.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220124,frontMatter:{}},l={},u=[{value:"Background :",id:"background-",level:3},{value:"Approach :",id:"approach-",level:3},{value:"Other Thoughts:",id:"other-thoughts",level:3},{value:"How do we transition to this new approach?",id:"how-do-we-transition-to-this-new-approach",level:3}],d={toc:u},c="wrapper";function p(e){let{components:t,...n}=e;return(0,o.kt)(c,(0,a.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Feature Name: Revisit Automated Dependency Resolution Logic"),(0,o.kt)("li",{parentName:"ul"},"Status: Draft"),(0,o.kt)("li",{parentName:"ul"},"Start Date: 2022-01-24"),(0,o.kt)("li",{parentName:"ul"},"Authors: ")),(0,o.kt)("h1",{id:"summary"},"Summary"),(0,o.kt)("p",null,"Optimus is a data warehouse management system, data is at the core of Optimus. Automated dependency resolution is the core problem which Optimus wants to address. The current windowing is confusing, so lets take a revisit to understand how we can solve this problem if we are tackling it fresh."),(0,o.kt)("h1",{id:"technical-design"},"Technical Design"),(0,o.kt)("h3",{id:"background-"},"Background :"),(0,o.kt)("p",null,"Input Data Flows through the system & it is expected to arrive at some delay or user gives enough buffer for all late data to arrive. Post that, the user expects to schedule the job to process the data after the max delay."),(0,o.kt)("p",null,"Keeping this basic idea in mind, what logic can be used to enable automated dependency resolution is the key question for us? And what all questions need to be answered for the same?"),(0,o.kt)("p",null,"Question 1 : What is the time range of data a job consumes from the primary sources?"),(0,o.kt)("p",null,"Question 2 : What is the time range of data a job writes?"),(0,o.kt)("p",null,"If these two questions be answered for every scheduled job then dependent jobs be computed accordingly."),(0,o.kt)("h3",{id:"approach-"},"Approach :"),(0,o.kt)("p",null,"Let's answer the ",(0,o.kt)("strong",{parentName:"p"},"Question 1"),", this is clearly a user input, there is no computation here. How intuitively a user input can be taken is the key here."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"data_window : \n max_delay : 1d/2h/1d2h\n amount : 1d/1w/2d/1m \n")),(0,o.kt)("p",null,(0,o.kt)("inlineCode",{parentName:"p"},"max_delay")," is used to identify the end time of the window."),(0,o.kt)("p",null,(0,o.kt)("inlineCode",{parentName:"p"},"amount")," is the amount of data the user is consuming from the primary sources."),(0,o.kt)("p",null,"Below is the current windowing configuration."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"}," window:\n size: 24h\n offset: 24h\n truncate_to: d\n")),(0,o.kt)("p",null,"Let's answer ",(0,o.kt)("strong",{parentName:"p"},"Question 2"),", I believe this is mainly linked with the schedule of the job, if the job is scheduled daily then the expectation is the job makes data available for a whole day, if hourly then for a whole hour, irrespective of the input_window. What exactly is the time range can be computed by ",(0,o.kt)("inlineCode",{parentName:"p"},"max_delay")," and ",(0,o.kt)("inlineCode",{parentName:"p"},"schedule_frequency")," . "),(0,o.kt)("p",null,"If the job has a max_delay of 2h & the job is scheduled at 2 AM UTC then the job is making the data available for the entire previous day, irrespective of the window(1d,1m,1w)."),(0,o.kt)("p",null,"WIth this core idea there are few scenarios which should not be allowed or which cannot be used for automated depedency resolution to work in those cases the jobs just depend on the previous jobs for eg., dynamic schedules. If a job is scheduled only 1st,2nd & 3rd hour of the day."),(0,o.kt)("p",null,"The next part of the solution is how to do the actual dependency resolution."),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},"Compute the data_window based on user input & the schedule time."),(0,o.kt)("li",{parentName:"ol"},"Identify all the upstreams."),(0,o.kt)("li",{parentName:"ol"},"Starting from the current schedule_time get each upstream's schedule in the past & future and compute the output data range till it finds runs which falls outside the window in each direction.")),(0,o.kt)("h3",{id:"other-thoughts"},"Other Thoughts:"),(0,o.kt)("p",null,"Inorder to keep things simple for most of the users, if a user doesn't define any window then the jobs depend on the immediate upstream by schedule_time for that job & as well as the jobs that are depending the above job."),(0,o.kt)("p",null,(0,o.kt)("inlineCode",{parentName:"p"},"dstart")," & ",(0,o.kt)("inlineCode",{parentName:"p"},"dend")," macros will still be offered to users which they can use for data filtering in their queries."),(0,o.kt)("h3",{id:"how-do-we-transition-to-this-new-approach"},"How do we transition to this new approach?"),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},"Support Old & New approaches both, migrate all jobs to the new strategy & later cleanup the code.")))}p.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2106],{3905:(e,t,n)=>{n.d(t,{Zo:()=>d,kt:()=>m});var a=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=a.createContext({}),u=function(e){var t=a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=u(e.components);return a.createElement(l.Provider,{value:t},e.children)},c="mdxType",p={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},h=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,l=e.parentName,d=s(e,["components","mdxType","originalType","parentName"]),c=u(n),h=o,m=c["".concat(l,".").concat(h)]||c[h]||p[h]||r;return n?a.createElement(m,i(i({ref:t},d),{},{components:n})):a.createElement(m,i({ref:t},d))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=h;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[c]="string"==typeof e?e:o,i[1]=s;for(var u=2;u{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>i,default:()=>p,frontMatter:()=>r,metadata:()=>s,toc:()=>u});var a=n(7462),o=(n(7294),n(3905));const r={},i=void 0,s={unversionedId:"rfcs/revisit_automated_dependency_resolution",id:"rfcs/revisit_automated_dependency_resolution",title:"revisit_automated_dependency_resolution",description:"- Feature Name: Revisit Automated Dependency Resolution Logic",source:"@site/docs/rfcs/20220124_revisit_automated_dependency_resolution.md",sourceDirName:"rfcs",slug:"/rfcs/revisit_automated_dependency_resolution",permalink:"/optimus/docs/rfcs/revisit_automated_dependency_resolution",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220124_revisit_automated_dependency_resolution.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220124,frontMatter:{}},l={},u=[{value:"Background :",id:"background-",level:3},{value:"Approach :",id:"approach-",level:3},{value:"Other Thoughts:",id:"other-thoughts",level:3},{value:"How do we transition to this new approach?",id:"how-do-we-transition-to-this-new-approach",level:3}],d={toc:u},c="wrapper";function p(e){let{components:t,...n}=e;return(0,o.kt)(c,(0,a.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Feature Name: Revisit Automated Dependency Resolution Logic"),(0,o.kt)("li",{parentName:"ul"},"Status: Draft"),(0,o.kt)("li",{parentName:"ul"},"Start Date: 2022-01-24"),(0,o.kt)("li",{parentName:"ul"},"Authors: ")),(0,o.kt)("h1",{id:"summary"},"Summary"),(0,o.kt)("p",null,"Optimus is a data warehouse management system, data is at the core of Optimus. Automated dependency resolution is the core problem which Optimus wants to address. The current windowing is confusing, so lets take a revisit to understand how we can solve this problem if we are tackling it fresh."),(0,o.kt)("h1",{id:"technical-design"},"Technical Design"),(0,o.kt)("h3",{id:"background-"},"Background :"),(0,o.kt)("p",null,"Input Data Flows through the system & it is expected to arrive at some delay or user gives enough buffer for all late data to arrive. Post that, the user expects to schedule the job to process the data after the max delay."),(0,o.kt)("p",null,"Keeping this basic idea in mind, what logic can be used to enable automated dependency resolution is the key question for us? And what all questions need to be answered for the same?"),(0,o.kt)("p",null,"Question 1 : What is the time range of data a job consumes from the primary sources?"),(0,o.kt)("p",null,"Question 2 : What is the time range of data a job writes?"),(0,o.kt)("p",null,"If these two questions be answered for every scheduled job then dependent jobs be computed accordingly."),(0,o.kt)("h3",{id:"approach-"},"Approach :"),(0,o.kt)("p",null,"Let's answer the ",(0,o.kt)("strong",{parentName:"p"},"Question 1"),", this is clearly a user input, there is no computation here. How intuitively a user input can be taken is the key here."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"data_window : \n max_delay : 1d/2h/1d2h\n amount : 1d/1w/2d/1m \n")),(0,o.kt)("p",null,(0,o.kt)("inlineCode",{parentName:"p"},"max_delay")," is used to identify the end time of the window."),(0,o.kt)("p",null,(0,o.kt)("inlineCode",{parentName:"p"},"amount")," is the amount of data the user is consuming from the primary sources."),(0,o.kt)("p",null,"Below is the current windowing configuration."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"}," window:\n size: 24h\n offset: 24h\n truncate_to: d\n")),(0,o.kt)("p",null,"Let's answer ",(0,o.kt)("strong",{parentName:"p"},"Question 2"),", I believe this is mainly linked with the schedule of the job, if the job is scheduled daily then the expectation is the job makes data available for a whole day, if hourly then for a whole hour, irrespective of the input_window. What exactly is the time range can be computed by ",(0,o.kt)("inlineCode",{parentName:"p"},"max_delay")," and ",(0,o.kt)("inlineCode",{parentName:"p"},"schedule_frequency")," . "),(0,o.kt)("p",null,"If the job has a max_delay of 2h & the job is scheduled at 2 AM UTC then the job is making the data available for the entire previous day, irrespective of the window(1d,1m,1w)."),(0,o.kt)("p",null,"WIth this core idea there are few scenarios which should not be allowed or which cannot be used for automated depedency resolution to work in those cases the jobs just depend on the previous jobs for eg., dynamic schedules. If a job is scheduled only 1st,2nd & 3rd hour of the day."),(0,o.kt)("p",null,"The next part of the solution is how to do the actual dependency resolution."),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},"Compute the data_window based on user input & the schedule time."),(0,o.kt)("li",{parentName:"ol"},"Identify all the upstreams."),(0,o.kt)("li",{parentName:"ol"},"Starting from the current schedule_time get each upstream's schedule in the past & future and compute the output data range till it finds runs which falls outside the window in each direction.")),(0,o.kt)("h3",{id:"other-thoughts"},"Other Thoughts:"),(0,o.kt)("p",null,"Inorder to keep things simple for most of the users, if a user doesn't define any window then the jobs depend on the immediate upstream by schedule_time for that job & as well as the jobs that are depending the above job."),(0,o.kt)("p",null,(0,o.kt)("inlineCode",{parentName:"p"},"dstart")," & ",(0,o.kt)("inlineCode",{parentName:"p"},"dend")," macros will still be offered to users which they can use for data filtering in their queries."),(0,o.kt)("h3",{id:"how-do-we-transition-to-this-new-approach"},"How do we transition to this new approach?"),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},"Support Old & New approaches both, migrate all jobs to the new strategy & later cleanup the code.")))}p.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/174ad3e8.2cd20389.js b/assets/js/174ad3e8.e7d3cc5f.js similarity index 99% rename from assets/js/174ad3e8.2cd20389.js rename to assets/js/174ad3e8.e7d3cc5f.js index 52bdbfaa71..3c2561adfd 100644 --- a/assets/js/174ad3e8.2cd20389.js +++ b/assets/js/174ad3e8.e7d3cc5f.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2389],{3905:(e,t,a)=>{a.d(t,{Zo:()=>d,kt:()=>h});var n=a(7294);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var s=n.createContext({}),p=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},d=function(e){var t=p(e.components);return n.createElement(s.Provider,{value:t},e.children)},c="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,s=e.parentName,d=l(e,["components","mdxType","originalType","parentName"]),c=p(a),m=o,h=c["".concat(s,".").concat(m)]||c[m]||u[m]||r;return a?n.createElement(h,i(i({ref:t},d),{},{components:a})):n.createElement(h,i({ref:t},d))}));function h(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,i=new Array(r);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[c]="string"==typeof e?e:o,i[1]=l;for(var p=2;p{a.r(t),a.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>u,frontMatter:()=>r,metadata:()=>l,toc:()=>p});var n=a(7462),o=(a(7294),a(3905));const r={},i="Create Job Specifications",l={unversionedId:"client-guide/create-job-specifications",id:"client-guide/create-job-specifications",title:"Create Job Specifications",description:"A Job is the fundamental execution unit of an Optimus data pipeline. It can be scheduled, configured and is always",source:"@site/docs/client-guide/create-job-specifications.md",sourceDirName:"client-guide",slug:"/client-guide/create-job-specifications",permalink:"/optimus/docs/client-guide/create-job-specifications",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/create-job-specifications.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Manage BigQuery Resource",permalink:"/optimus/docs/client-guide/manage-bigquery-resource"},next:{title:"Setting up Alert to Job",permalink:"/optimus/docs/client-guide/setting-up-alert"}},s={},p=[{value:"Initialize Job Specification",id:"initialize-job-specification",level:2},{value:"Understanding the Job Specifications",id:"understanding-the-job-specifications",level:2},{value:"Behavior",id:"behavior",level:3},{value:"Task",id:"task",level:3},{value:"Dependencies",id:"dependencies",level:3},{value:"Metadata",id:"metadata",level:3},{value:"Completing the Transformation Task",id:"completing-the-transformation-task",level:2},{value:"Adding Hook",id:"adding-hook",level:3}],d={toc:p},c="wrapper";function u(e){let{components:t,...r}=e;return(0,o.kt)(c,(0,n.Z)({},d,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"create-job-specifications"},"Create Job Specifications"),(0,o.kt)("p",null,"A Job is the fundamental execution unit of an Optimus data pipeline. It can be scheduled, configured and is always\nmapped to a single transformation type (eg, BQ-to-BQ, GCS-to-BQ, etc). It can have dependencies over other jobs and\nshould only execute once the dependent job is successfully completed."),(0,o.kt)("p",null,"A job can also be configured with Hooks as part of its lifecycle, which can be triggered before or after the job.\nPlease go through the ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/job"},"concept")," to know more about it."),(0,o.kt)("p",null,"Before we begin, let\u2019s understand the flow of job creation & deployment (later) in Optimus."),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"Create Job Flow",src:a(5591).Z,title:"CreateJobSpecFlow",width:"1600",height:"979"})),(0,o.kt)("p",null,'For this guide, we\'ll be creating a job that writes "hello YYYY-MM-DD" to a table every day at 03.00 AM. We\'ll use the\nBQ-to-BQ transformation type. For the purpose of this guide, we\'ll assume that the Google Cloud Project name is\n"sample-project" & dataset is just called "playground".'),(0,o.kt)("h2",{id:"initialize-job-specification"},"Initialize Job Specification"),(0,o.kt)("p",null,"Open your terminal and create a new directory that will hold the specifications created by Optimus CLI. Once ready,\nyou can run the following command and answer the corresponding prompts (do note that some prompts would be to select\nfrom options instead of input):"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job create \n? Please choose the namespace: sample_namespace\n? Provide new directory name to create for this spec? [.] sample-project.playground.table1\n? What is the job name? sample-project.playground.table1\n? Who is the owner of this job? sample_owner\n? Select task to run? bq2bq\n? Specify the schedule start date 2023-01-26\n? Specify the schedule interval (in crontab notation) 0 2 * * *\n? Window truncate to: d\n? Window offset: 0\n? Window size: 24h\n? Project ID sample-project\n? Dataset Name playground\n? Table ID table1\n? Load method to use on destination REPLACE\nJob successfully created at sample-project.playground.table1\n")),(0,o.kt)("p",null,(0,o.kt)("em",{parentName:"p"},"Note: window configuration option may be different, depending on the provided presets under project configuration ",(0,o.kt)("a",{parentName:"em",href:"/optimus/docs/concepts/intervals-and-windows"},"reference"))),(0,o.kt)("p",null,"After running the job create command, the job specification file and assets directory are created in the following directory."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n| \u2514\u2500\u2500 sample-project.playground.table1\n| \u2514\u2500\u2500 assets\n| \u2514\u2500\u2500 query.sql\n| \u2514\u2500\u2500 job.yaml\n\u2502 \u2514\u2500\u2500 resources\n\u2514\u2500\u2500 optimus.yaml\n")),(0,o.kt)("p",null,"Do notice that query.sql file is also generated. This is because, for BQ to BQ job, transformation logic lies in the\nquery.sql file. We will update this file based on the requirement later."),(0,o.kt)("p",null,"For now, let\u2019s take a deeper look at the job.yaml that Optimus has generated and understands what it does. After\ntaking a look at the possible configurations, we will try to complete the transformation task and take a look at how\nto add a hook."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground.table1\nowner: sample_owner\nschedule:\n start_date: "2023-01-26"\n interval: 0 2 * * *\nbehavior:\n depends_on_past: false\ntask:\n name: bq2bq\n config:\n DATASET: playground\n LOAD_METHOD: REPLACE\n PROJECT: sample-project\n SQL_TYPE: STANDARD\n TABLE: table1\n window:\n size: 24h\n offset: "0"\n truncate_to: d\nlabels:\n orchestrator: optimus\nhooks: []\ndependencies: []\n')),(0,o.kt)("p",null,"Note that if presets are specified under project, then ",(0,o.kt)("inlineCode",{parentName:"p"},"window")," may contains only the name of the presets, like the following example:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"...\ntask:\n ...\n window:\n preset: yesterday\n...\n")),(0,o.kt)("p",null,"For more detail about window configuration and preset, please check ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/intervals-and-windows"},"this page"),"."),(0,o.kt)("h2",{id:"understanding-the-job-specifications"},"Understanding the Job Specifications"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Job Configuration"),(0,o.kt)("th",{parentName:"tr",align:null},"Description"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Version"),(0,o.kt)("td",{parentName:"tr",align:null},"Version 1 and 2 (recommended) are available. This affects the window version to be used. Note that presets always use window v2")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Name"),(0,o.kt)("td",{parentName:"tr",align:null},"Should be unique in the project.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Owner"),(0,o.kt)("td",{parentName:"tr",align:null},"Owner of the job, can be an email, team name, slack handle, or anything that works for your team.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Schedule"),(0,o.kt)("td",{parentName:"tr",align:null},"Specifications needed to schedule a job, such as start_date, end_date and interval (cron)")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Behavior"),(0,o.kt)("td",{parentName:"tr",align:null},"Specifications that represents how the scheduled jobs should behave, for example when the run is failed.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Task"),(0,o.kt)("td",{parentName:"tr",align:null},"Specifications related to the transformation task")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Hooks"),(0,o.kt)("td",{parentName:"tr",align:null},"Name & configuration of pre/post hooks. Take a look at how to add hooks ",(0,o.kt)("a",{parentName:"td",href:"#adding-hook"},"here"),".")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Labels"),(0,o.kt)("td",{parentName:"tr",align:null},"Help you to identify your job. Any of the values will also be marked as a tag in Airflow.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Dependencies"),(0,o.kt)("td",{parentName:"tr",align:null},"Represent the list of jobs that are considered upstream.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Metadata"),(0,o.kt)("td",{parentName:"tr",align:null},"Represents additional resource and scheduler configurations.")))),(0,o.kt)("h3",{id:"behavior"},"Behavior"),(0,o.kt)("p",null,"Behavior specification might consist:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"depends_on_past: set to true to not allow the task to run, if the previous task run has not been succeeded yet"),(0,o.kt)("li",{parentName:"ul"},"retry",(0,o.kt)("ul",{parentName:"li"},(0,o.kt)("li",{parentName:"ul"},"count: represents how many times it will try to retrigger the job if the job failed to run "),(0,o.kt)("li",{parentName:"ul"},"delay"),(0,o.kt)("li",{parentName:"ul"},"exponential_backoff"))),(0,o.kt)("li",{parentName:"ul"},"notify: Alert configurations. Take a look more at this ",(0,o.kt)("a",{parentName:"li",href:"/optimus/docs/client-guide/setting-up-alert"},"here"),".")),(0,o.kt)("h3",{id:"task"},"Task"),(0,o.kt)("p",null,"Task specification might consist:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"name"),(0,o.kt)("li",{parentName:"ul"},"config: Some configs might be needed for a specific task type. For example, for BQ to BQ task, it is required to have\nBQ_SERVICE_ACCOUNT, PROJECT, DATASET, TABLE, SQL_TYPE, LOAD_METHOD configs. Take a look at the details of what is load method here."),(0,o.kt)("li",{parentName:"ul"},"window: Take a look at the details of the window ",(0,o.kt)("a",{parentName:"li",href:"/optimus/docs/concepts/intervals-and-windows"},"here"),".")),(0,o.kt)("h3",{id:"dependencies"},"Dependencies"),(0,o.kt)("p",null,"Represent the list of jobs that are considered upstream."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"If the job is in a different project, include the Optimus\u2019 project name in the prefix."),(0,o.kt)("li",{parentName:"ul"},"If the job is in the same project, simply mentioning the job name is sufficient.")),(0,o.kt)("p",null,"Example:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"dependencies:\n- job: sample-project.playground.table1\n- job: other-project/other-project.playground.table2\n")),(0,o.kt)("h3",{id:"metadata"},"Metadata"),(0,o.kt)("p",null,"Below specifications can be set in Metadata section:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"resource"),": set up CPU/memory request/limit"),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"airflow"),": set up which Airflow pool and what is the queue configuration for this job")),(0,o.kt)("h2",{id:"completing-the-transformation-task"},"Completing the Transformation Task"),(0,o.kt)("p",null,"Let\u2019s retake a look at the generated task specifications"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},'task:\n name: bq2bq\n config:\n DATASET: playground\n LOAD_METHOD: REPLACE\n PROJECT: sample-project\n SQL_TYPE: STANDARD\n TABLE: table1\n window:\n size: 24h\n offset: "0"\n truncate_to: d\n')),(0,o.kt)("p",null,"Here are the details of each configuration and the allowed values:"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Config Name"),(0,o.kt)("th",{parentName:"tr",align:null},"Description"),(0,o.kt)("th",{parentName:"tr",align:null},"Values"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"PROJECT"),(0,o.kt)("td",{parentName:"tr",align:null},"GCP project ID of the destination BigQuery table"),(0,o.kt)("td",{parentName:"tr",align:null})),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"DATASET"),(0,o.kt)("td",{parentName:"tr",align:null},"BigQuery dataset name of the destination table"),(0,o.kt)("td",{parentName:"tr",align:null})),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"TABLE"),(0,o.kt)("td",{parentName:"tr",align:null},"the table name of the destination table"),(0,o.kt)("td",{parentName:"tr",align:null})),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"LOAD_METHOD"),(0,o.kt)("td",{parentName:"tr",align:null},"method to load data to the destination tables"),(0,o.kt)("td",{parentName:"tr",align:null},"Take a detailed look ",(0,o.kt)("a",{parentName:"td",href:"https://github.com/goto/transformers/blob/main/task/bq2bq/README.md"},"here"))),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"PARTITION_FILTER"),(0,o.kt)("td",{parentName:"tr",align:null},"Used to identify target partitions to replace in a REPLACE query. This can be left empty and Optimus will figure out the target partitions automatically but it's cheaper and faster to specify the condition. This filter will be used as a where clause in a merge statement to delete the partitions from the destination table."),(0,o.kt)("td",{parentName:"tr",align:null},'event_timestamp >= "{{.DSTART}}" AND event_timestamp < "{{.DEND}}"')))),(0,o.kt)("p",null,"Now let's try to modify the core transformation logic that lies in ",(0,o.kt)("inlineCode",{parentName:"p"},"assets/query.sql"),'. Remember that we are going to\ncreate a job that writes "hello YYYY-MM-DD" to a table every day at 03.00 AM. Now, let\u2019s modify the query so it prints\nwhat we intended:'),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},'SELECT CONCAT("Hello, ", "{{.DEND}}") AS message;\n')),(0,o.kt)("p",null,"{{.DEND}} is a macro that is replaced with the current execution date (in YYYY-MM-DD format) of the task (\nnote that this is the execution date of when the task was supposed to run, not when it actually runs). Take a detailed\nlook at the supported macros ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/macros"},"here"),"."),(0,o.kt)("p",null,"Do notice that the query is not sourcing from any other table. This means the job we are creating will not have any\n",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/dependency"},"dependency")," unless we manually specify so in the job specification YAML file.\nHowever, if for any reason you are querying from another resource and want to ignore the dependency, add\n",(0,o.kt)("inlineCode",{parentName:"p"},"@ignoreupstream")," annotation just before the table name, for example:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},"SELECT column1, column2, column3\nFROM `sample-project.playground.source1` s1\nLEFT JOIN /* @ignoreupstream */\n`sample-project.playground.source2` s2\nON (s1.id = s2.s1_id)\nWHERE\nDATE(`load_timestamp`) >= DATE('{{.DSTART}}')\nAND DATE(`load_timestamp`) < DATE('{{.DEND}}');\n")),(0,o.kt)("h3",{id:"adding-hook"},"Adding Hook"),(0,o.kt)("p",null,"There might be a certain operation that you might want to run before or after the Job. Please go through the\n",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/job"},"concept")," to know more about it."),(0,o.kt)("p",null,"For this guide, let\u2019s add a post hook that will audit our BigQuery data using ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/predator"},"Predator"),".\nYou can find the Predator plugin YAML file ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/predator/blob/main/optimus-plugin-predator.yaml"},"here"),"\nand have the plugin installed in your ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/server-guide/installing-plugins"},"server")," and ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/installing-plugin"},"client"),"."),(0,o.kt)("p",null,"In order to add a hook to an existing Job, run the following command and answer the corresponding prompts:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-shell"},'$ optimus job addhook\n? Please choose the namespace: sample_namespace\n? Select a Job sample-project.playground.table1\n? Filter expression for extracting transformation rows? __PARTITION__ >= date("{{ .DSTART | Date }}") AND __PARTITION__ < date("{{ .DEND | Date }}")\n? Specify the profile/audit result grouping field (empty to not group the result)\n? Choose the profiling mode complete\n\nHook successfully added to sample-project.playground.table1\n')),(0,o.kt)("p",null,"With the above prompt, we're adding the predator hook post the execution of the primary job. Filter expression\nconfiguration and the rest of the questions are specific to a predator hook, and it might be different for other hooks."),(0,o.kt)("p",null,"After this, existing job.yaml file will get updated with the new hook config."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"hooks:\n- name: predator\n config:\n AUDIT_TIME: '{{.EXECUTION_TIME}}'\n BQ_DATASET: '{{.TASK__DATASET}}'\n BQ_PROJECT: '{{.TASK__PROJECT}}'\n BQ_TABLE: '{{.TASK__TABLE}}'\n FILTER: __PARTITION__ >= date(\"{{ .DSTART | Date }}\") AND __PARTITION__ < date(\"{{ .DEND | Date }}\")\n GROUP: \"\"\n MODE: complete\n PREDATOR_URL: '{{.GLOBAL__PREDATOR_HOST}}'\n SUB_COMMAND: profile_audit\n")))}u.isMDXComponent=!0},5591:(e,t,a)=>{a.d(t,{Z:()=>n});const n=a.p+"assets/images/CreateJobSpecFlow-8a5b023871779f63e58da0ab2455770f.png"}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2389],{3905:(e,t,a)=>{a.d(t,{Zo:()=>d,kt:()=>h});var n=a(7294);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var s=n.createContext({}),p=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},d=function(e){var t=p(e.components);return n.createElement(s.Provider,{value:t},e.children)},c="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,s=e.parentName,d=l(e,["components","mdxType","originalType","parentName"]),c=p(a),m=o,h=c["".concat(s,".").concat(m)]||c[m]||u[m]||r;return a?n.createElement(h,i(i({ref:t},d),{},{components:a})):n.createElement(h,i({ref:t},d))}));function h(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,i=new Array(r);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[c]="string"==typeof e?e:o,i[1]=l;for(var p=2;p{a.r(t),a.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>u,frontMatter:()=>r,metadata:()=>l,toc:()=>p});var n=a(7462),o=(a(7294),a(3905));const r={},i="Create Job Specifications",l={unversionedId:"client-guide/create-job-specifications",id:"client-guide/create-job-specifications",title:"Create Job Specifications",description:"A Job is the fundamental execution unit of an Optimus data pipeline. It can be scheduled, configured and is always",source:"@site/docs/client-guide/create-job-specifications.md",sourceDirName:"client-guide",slug:"/client-guide/create-job-specifications",permalink:"/optimus/docs/client-guide/create-job-specifications",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/create-job-specifications.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Manage BigQuery Resource",permalink:"/optimus/docs/client-guide/manage-bigquery-resource"},next:{title:"Setting up Alert to Job",permalink:"/optimus/docs/client-guide/setting-up-alert"}},s={},p=[{value:"Initialize Job Specification",id:"initialize-job-specification",level:2},{value:"Understanding the Job Specifications",id:"understanding-the-job-specifications",level:2},{value:"Behavior",id:"behavior",level:3},{value:"Task",id:"task",level:3},{value:"Dependencies",id:"dependencies",level:3},{value:"Metadata",id:"metadata",level:3},{value:"Completing the Transformation Task",id:"completing-the-transformation-task",level:2},{value:"Adding Hook",id:"adding-hook",level:3}],d={toc:p},c="wrapper";function u(e){let{components:t,...r}=e;return(0,o.kt)(c,(0,n.Z)({},d,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"create-job-specifications"},"Create Job Specifications"),(0,o.kt)("p",null,"A Job is the fundamental execution unit of an Optimus data pipeline. It can be scheduled, configured and is always\nmapped to a single transformation type (eg, BQ-to-BQ, GCS-to-BQ, etc). It can have dependencies over other jobs and\nshould only execute once the dependent job is successfully completed."),(0,o.kt)("p",null,"A job can also be configured with Hooks as part of its lifecycle, which can be triggered before or after the job.\nPlease go through the ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/job"},"concept")," to know more about it."),(0,o.kt)("p",null,"Before we begin, let\u2019s understand the flow of job creation & deployment (later) in Optimus."),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"Create Job Flow",src:a(5591).Z,title:"CreateJobSpecFlow",width:"1600",height:"979"})),(0,o.kt)("p",null,'For this guide, we\'ll be creating a job that writes "hello YYYY-MM-DD" to a table every day at 03.00 AM. We\'ll use the\nBQ-to-BQ transformation type. For the purpose of this guide, we\'ll assume that the Google Cloud Project name is\n"sample-project" & dataset is just called "playground".'),(0,o.kt)("h2",{id:"initialize-job-specification"},"Initialize Job Specification"),(0,o.kt)("p",null,"Open your terminal and create a new directory that will hold the specifications created by Optimus CLI. Once ready,\nyou can run the following command and answer the corresponding prompts (do note that some prompts would be to select\nfrom options instead of input):"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job create \n? Please choose the namespace: sample_namespace\n? Provide new directory name to create for this spec? [.] sample-project.playground.table1\n? What is the job name? sample-project.playground.table1\n? Who is the owner of this job? sample_owner\n? Select task to run? bq2bq\n? Specify the schedule start date 2023-01-26\n? Specify the schedule interval (in crontab notation) 0 2 * * *\n? Window truncate to: d\n? Window offset: 0\n? Window size: 24h\n? Project ID sample-project\n? Dataset Name playground\n? Table ID table1\n? Load method to use on destination REPLACE\nJob successfully created at sample-project.playground.table1\n")),(0,o.kt)("p",null,(0,o.kt)("em",{parentName:"p"},"Note: window configuration option may be different, depending on the provided presets under project configuration ",(0,o.kt)("a",{parentName:"em",href:"/optimus/docs/concepts/intervals-and-windows"},"reference"))),(0,o.kt)("p",null,"After running the job create command, the job specification file and assets directory are created in the following directory."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n| \u2514\u2500\u2500 sample-project.playground.table1\n| \u2514\u2500\u2500 assets\n| \u2514\u2500\u2500 query.sql\n| \u2514\u2500\u2500 job.yaml\n\u2502 \u2514\u2500\u2500 resources\n\u2514\u2500\u2500 optimus.yaml\n")),(0,o.kt)("p",null,"Do notice that query.sql file is also generated. This is because, for BQ to BQ job, transformation logic lies in the\nquery.sql file. We will update this file based on the requirement later."),(0,o.kt)("p",null,"For now, let\u2019s take a deeper look at the job.yaml that Optimus has generated and understands what it does. After\ntaking a look at the possible configurations, we will try to complete the transformation task and take a look at how\nto add a hook."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground.table1\nowner: sample_owner\nschedule:\n start_date: "2023-01-26"\n interval: 0 2 * * *\nbehavior:\n depends_on_past: false\ntask:\n name: bq2bq\n config:\n DATASET: playground\n LOAD_METHOD: REPLACE\n PROJECT: sample-project\n SQL_TYPE: STANDARD\n TABLE: table1\n window:\n size: 24h\n offset: "0"\n truncate_to: d\nlabels:\n orchestrator: optimus\nhooks: []\ndependencies: []\n')),(0,o.kt)("p",null,"Note that if presets are specified under project, then ",(0,o.kt)("inlineCode",{parentName:"p"},"window")," may contains only the name of the presets, like the following example:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"...\ntask:\n ...\n window:\n preset: yesterday\n...\n")),(0,o.kt)("p",null,"For more detail about window configuration and preset, please check ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/intervals-and-windows"},"this page"),"."),(0,o.kt)("h2",{id:"understanding-the-job-specifications"},"Understanding the Job Specifications"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Job Configuration"),(0,o.kt)("th",{parentName:"tr",align:null},"Description"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Version"),(0,o.kt)("td",{parentName:"tr",align:null},"Version 1 and 2 (recommended) are available. This affects the window version to be used. Note that presets always use window v2")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Name"),(0,o.kt)("td",{parentName:"tr",align:null},"Should be unique in the project.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Owner"),(0,o.kt)("td",{parentName:"tr",align:null},"Owner of the job, can be an email, team name, slack handle, or anything that works for your team.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Schedule"),(0,o.kt)("td",{parentName:"tr",align:null},"Specifications needed to schedule a job, such as start_date, end_date and interval (cron)")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Behavior"),(0,o.kt)("td",{parentName:"tr",align:null},"Specifications that represents how the scheduled jobs should behave, for example when the run is failed.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Task"),(0,o.kt)("td",{parentName:"tr",align:null},"Specifications related to the transformation task")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Hooks"),(0,o.kt)("td",{parentName:"tr",align:null},"Name & configuration of pre/post hooks. Take a look at how to add hooks ",(0,o.kt)("a",{parentName:"td",href:"#adding-hook"},"here"),".")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Labels"),(0,o.kt)("td",{parentName:"tr",align:null},"Help you to identify your job. Any of the values will also be marked as a tag in Airflow.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Dependencies"),(0,o.kt)("td",{parentName:"tr",align:null},"Represent the list of jobs that are considered upstream.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Metadata"),(0,o.kt)("td",{parentName:"tr",align:null},"Represents additional resource and scheduler configurations.")))),(0,o.kt)("h3",{id:"behavior"},"Behavior"),(0,o.kt)("p",null,"Behavior specification might consist:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"depends_on_past: set to true to not allow the task to run, if the previous task run has not been succeeded yet"),(0,o.kt)("li",{parentName:"ul"},"retry",(0,o.kt)("ul",{parentName:"li"},(0,o.kt)("li",{parentName:"ul"},"count: represents how many times it will try to retrigger the job if the job failed to run "),(0,o.kt)("li",{parentName:"ul"},"delay"),(0,o.kt)("li",{parentName:"ul"},"exponential_backoff"))),(0,o.kt)("li",{parentName:"ul"},"notify: Alert configurations. Take a look more at this ",(0,o.kt)("a",{parentName:"li",href:"/optimus/docs/client-guide/setting-up-alert"},"here"),".")),(0,o.kt)("h3",{id:"task"},"Task"),(0,o.kt)("p",null,"Task specification might consist:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"name"),(0,o.kt)("li",{parentName:"ul"},"config: Some configs might be needed for a specific task type. For example, for BQ to BQ task, it is required to have\nBQ_SERVICE_ACCOUNT, PROJECT, DATASET, TABLE, SQL_TYPE, LOAD_METHOD configs. Take a look at the details of what is load method here."),(0,o.kt)("li",{parentName:"ul"},"window: Take a look at the details of the window ",(0,o.kt)("a",{parentName:"li",href:"/optimus/docs/concepts/intervals-and-windows"},"here"),".")),(0,o.kt)("h3",{id:"dependencies"},"Dependencies"),(0,o.kt)("p",null,"Represent the list of jobs that are considered upstream."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"If the job is in a different project, include the Optimus\u2019 project name in the prefix."),(0,o.kt)("li",{parentName:"ul"},"If the job is in the same project, simply mentioning the job name is sufficient.")),(0,o.kt)("p",null,"Example:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"dependencies:\n- job: sample-project.playground.table1\n- job: other-project/other-project.playground.table2\n")),(0,o.kt)("h3",{id:"metadata"},"Metadata"),(0,o.kt)("p",null,"Below specifications can be set in Metadata section:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"resource"),": set up CPU/memory request/limit"),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"airflow"),": set up which Airflow pool and what is the queue configuration for this job")),(0,o.kt)("h2",{id:"completing-the-transformation-task"},"Completing the Transformation Task"),(0,o.kt)("p",null,"Let\u2019s retake a look at the generated task specifications"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},'task:\n name: bq2bq\n config:\n DATASET: playground\n LOAD_METHOD: REPLACE\n PROJECT: sample-project\n SQL_TYPE: STANDARD\n TABLE: table1\n window:\n size: 24h\n offset: "0"\n truncate_to: d\n')),(0,o.kt)("p",null,"Here are the details of each configuration and the allowed values:"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Config Name"),(0,o.kt)("th",{parentName:"tr",align:null},"Description"),(0,o.kt)("th",{parentName:"tr",align:null},"Values"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"PROJECT"),(0,o.kt)("td",{parentName:"tr",align:null},"GCP project ID of the destination BigQuery table"),(0,o.kt)("td",{parentName:"tr",align:null})),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"DATASET"),(0,o.kt)("td",{parentName:"tr",align:null},"BigQuery dataset name of the destination table"),(0,o.kt)("td",{parentName:"tr",align:null})),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"TABLE"),(0,o.kt)("td",{parentName:"tr",align:null},"the table name of the destination table"),(0,o.kt)("td",{parentName:"tr",align:null})),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"LOAD_METHOD"),(0,o.kt)("td",{parentName:"tr",align:null},"method to load data to the destination tables"),(0,o.kt)("td",{parentName:"tr",align:null},"Take a detailed look ",(0,o.kt)("a",{parentName:"td",href:"https://github.com/goto/transformers/blob/main/task/bq2bq/README.md"},"here"))),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"PARTITION_FILTER"),(0,o.kt)("td",{parentName:"tr",align:null},"Used to identify target partitions to replace in a REPLACE query. This can be left empty and Optimus will figure out the target partitions automatically but it's cheaper and faster to specify the condition. This filter will be used as a where clause in a merge statement to delete the partitions from the destination table."),(0,o.kt)("td",{parentName:"tr",align:null},'event_timestamp >= "{{.DSTART}}" AND event_timestamp < "{{.DEND}}"')))),(0,o.kt)("p",null,"Now let's try to modify the core transformation logic that lies in ",(0,o.kt)("inlineCode",{parentName:"p"},"assets/query.sql"),'. Remember that we are going to\ncreate a job that writes "hello YYYY-MM-DD" to a table every day at 03.00 AM. Now, let\u2019s modify the query so it prints\nwhat we intended:'),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},'SELECT CONCAT("Hello, ", "{{.DEND}}") AS message;\n')),(0,o.kt)("p",null,"{{.DEND}} is a macro that is replaced with the current execution date (in YYYY-MM-DD format) of the task (\nnote that this is the execution date of when the task was supposed to run, not when it actually runs). Take a detailed\nlook at the supported macros ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/macros"},"here"),"."),(0,o.kt)("p",null,"Do notice that the query is not sourcing from any other table. This means the job we are creating will not have any\n",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/dependency"},"dependency")," unless we manually specify so in the job specification YAML file.\nHowever, if for any reason you are querying from another resource and want to ignore the dependency, add\n",(0,o.kt)("inlineCode",{parentName:"p"},"@ignoreupstream")," annotation just before the table name, for example:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},"SELECT column1, column2, column3\nFROM `sample-project.playground.source1` s1\nLEFT JOIN /* @ignoreupstream */\n`sample-project.playground.source2` s2\nON (s1.id = s2.s1_id)\nWHERE\nDATE(`load_timestamp`) >= DATE('{{.DSTART}}')\nAND DATE(`load_timestamp`) < DATE('{{.DEND}}');\n")),(0,o.kt)("h3",{id:"adding-hook"},"Adding Hook"),(0,o.kt)("p",null,"There might be a certain operation that you might want to run before or after the Job. Please go through the\n",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/job"},"concept")," to know more about it."),(0,o.kt)("p",null,"For this guide, let\u2019s add a post hook that will audit our BigQuery data using ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/predator"},"Predator"),".\nYou can find the Predator plugin YAML file ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/predator/blob/main/optimus-plugin-predator.yaml"},"here"),"\nand have the plugin installed in your ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/server-guide/installing-plugins"},"server")," and ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/installing-plugin"},"client"),"."),(0,o.kt)("p",null,"In order to add a hook to an existing Job, run the following command and answer the corresponding prompts:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-shell"},'$ optimus job addhook\n? Please choose the namespace: sample_namespace\n? Select a Job sample-project.playground.table1\n? Filter expression for extracting transformation rows? __PARTITION__ >= date("{{ .DSTART | Date }}") AND __PARTITION__ < date("{{ .DEND | Date }}")\n? Specify the profile/audit result grouping field (empty to not group the result)\n? Choose the profiling mode complete\n\nHook successfully added to sample-project.playground.table1\n')),(0,o.kt)("p",null,"With the above prompt, we're adding the predator hook post the execution of the primary job. Filter expression\nconfiguration and the rest of the questions are specific to a predator hook, and it might be different for other hooks."),(0,o.kt)("p",null,"After this, existing job.yaml file will get updated with the new hook config."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"hooks:\n- name: predator\n config:\n AUDIT_TIME: '{{.EXECUTION_TIME}}'\n BQ_DATASET: '{{.TASK__DATASET}}'\n BQ_PROJECT: '{{.TASK__PROJECT}}'\n BQ_TABLE: '{{.TASK__TABLE}}'\n FILTER: __PARTITION__ >= date(\"{{ .DSTART | Date }}\") AND __PARTITION__ < date(\"{{ .DEND | Date }}\")\n GROUP: \"\"\n MODE: complete\n PREDATOR_URL: '{{.GLOBAL__PREDATOR_HOST}}'\n SUB_COMMAND: profile_audit\n")))}u.isMDXComponent=!0},5591:(e,t,a)=>{a.d(t,{Z:()=>n});const n=a.p+"assets/images/CreateJobSpecFlow-8a5b023871779f63e58da0ab2455770f.png"}}]); \ No newline at end of file diff --git a/assets/js/239f3910.29ef26a9.js b/assets/js/239f3910.f3153671.js similarity index 98% rename from assets/js/239f3910.29ef26a9.js rename to assets/js/239f3910.f3153671.js index 787b319626..dbd8572ec4 100644 --- a/assets/js/239f3910.29ef26a9.js +++ b/assets/js/239f3910.f3153671.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9887],{3905:(e,t,n)=>{n.d(t,{Zo:()=>l,kt:()=>d});var o=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function a(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=o.createContext({}),u=function(e){var t=o.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},l=function(e){var t=u(e.components);return o.createElement(s.Provider,{value:t},e.children)},p="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},f=o.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),p=u(n),f=r,d=p["".concat(s,".").concat(f)]||p[f]||m[f]||i;return n?o.createElement(d,a(a({ref:t},l),{},{components:n})):o.createElement(d,a({ref:t},l))}));function d(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=f;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c[p]="string"==typeof e?e:r,a[1]=c;for(var u=2;u{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>a,default:()=>m,frontMatter:()=>i,metadata:()=>c,toc:()=>u});var o=n(7462),r=(n(7294),n(3905));const i={},a="Contributing",c={unversionedId:"contribute/contribution-process",id:"contribute/contribution-process",title:"Contributing",description:"First off, thanks for taking the time to contribute! \ud83c\udf1f\ud83e\udd73",source:"@site/docs/contribute/contribution-process.md",sourceDirName:"contribute",slug:"/contribute/contribution-process",permalink:"/optimus/docs/contribute/contribution-process",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/contribute/contribution-process.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Tutorial of Plugin Development",permalink:"/optimus/docs/building-plugin/tutorial"},next:{title:"Developer Environment Setup",permalink:"/optimus/docs/contribute/developer-env-setup"}},s={},u=[{value:"Best practices",id:"best-practices",level:2},{value:"Code of Conduct",id:"code-of-conduct",level:2}],l={toc:u},p="wrapper";function m(e){let{components:t,...n}=e;return(0,r.kt)(p,(0,o.Z)({},l,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"contributing"},"Contributing"),(0,r.kt)("p",null,"First off, thanks for taking the time to contribute! \ud83c\udf1f\ud83e\udd73"),(0,r.kt)("p",null,"Before start contributing, feel free to ask questions or initiate conversation via GitHub discussion.\nYou are also welcome to create issue if you encounter a bug or to suggest feature enhancements."),(0,r.kt)("p",null,"Please note we have a code of conduct, please follow it in all your interactions with the project."),(0,r.kt)("h2",{id:"best-practices"},"Best practices"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Follow the ",(0,r.kt)("a",{parentName:"li",href:"https://www.conventionalcommits.org/en/v1.0.0/"},"conventional commit")," format for all commit messages."),(0,r.kt)("li",{parentName:"ul"},"Link the PR with the issue. This is mandatory to ensure there is sufficient information for the reviewer to understand\nyour PR."),(0,r.kt)("li",{parentName:"ul"},"When you make a PR for small change (such as fixing a typo, style change, or grammar fix), please squash your commits\nso that we can maintain a cleaner git history."),(0,r.kt)("li",{parentName:"ul"},"Docs live in the code repo under ",(0,r.kt)("a",{parentName:"li",href:"https://github.com/goto/optimus/tree/main/docs"},"docs"),". Please maintain the docs\nand any docs changes can be done in the same PR."),(0,r.kt)("li",{parentName:"ul"},"Avoid force-pushing as it makes reviewing difficult.")),(0,r.kt)("h2",{id:"code-of-conduct"},"Code of Conduct"),(0,r.kt)("p",null,"Examples of behavior that contributes to creating a positive environment include:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Using welcoming and inclusive language"),(0,r.kt)("li",{parentName:"ul"},"Being respectful of differing viewpoints and experiences"),(0,r.kt)("li",{parentName:"ul"},"Gracefully accepting constructive criticism"),(0,r.kt)("li",{parentName:"ul"},"Focusing on what is best for the project")),(0,r.kt)("p",null,"Things to keep in mind before creating a new commit:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Go through the project code conventions."),(0,r.kt)("li",{parentName:"ul"},"Commit ",(0,r.kt)("a",{parentName:"li",href:"https://www.conventionalcommits.org/en/v1.0.0/"},"guidelines")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("a",{parentName:"li",href:"https://github.com/cncf/foundation/blob/master/code-of-conduct.md"},"CNCF Code of Conduct"))))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9887],{3905:(e,t,n)=>{n.d(t,{Zo:()=>l,kt:()=>d});var o=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function a(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=o.createContext({}),u=function(e){var t=o.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},l=function(e){var t=u(e.components);return o.createElement(s.Provider,{value:t},e.children)},p="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},f=o.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,s=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),p=u(n),f=r,d=p["".concat(s,".").concat(f)]||p[f]||m[f]||i;return n?o.createElement(d,a(a({ref:t},l),{},{components:n})):o.createElement(d,a({ref:t},l))}));function d(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=f;var c={};for(var s in t)hasOwnProperty.call(t,s)&&(c[s]=t[s]);c.originalType=e,c[p]="string"==typeof e?e:r,a[1]=c;for(var u=2;u{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>a,default:()=>m,frontMatter:()=>i,metadata:()=>c,toc:()=>u});var o=n(7462),r=(n(7294),n(3905));const i={},a="Contributing",c={unversionedId:"contribute/contribution-process",id:"contribute/contribution-process",title:"Contributing",description:"First off, thanks for taking the time to contribute! \ud83c\udf1f\ud83e\udd73",source:"@site/docs/contribute/contribution-process.md",sourceDirName:"contribute",slug:"/contribute/contribution-process",permalink:"/optimus/docs/contribute/contribution-process",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/contribute/contribution-process.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Tutorial of Plugin Development",permalink:"/optimus/docs/building-plugin/tutorial"},next:{title:"Developer Environment Setup",permalink:"/optimus/docs/contribute/developer-env-setup"}},s={},u=[{value:"Best practices",id:"best-practices",level:2},{value:"Code of Conduct",id:"code-of-conduct",level:2}],l={toc:u},p="wrapper";function m(e){let{components:t,...n}=e;return(0,r.kt)(p,(0,o.Z)({},l,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"contributing"},"Contributing"),(0,r.kt)("p",null,"First off, thanks for taking the time to contribute! \ud83c\udf1f\ud83e\udd73"),(0,r.kt)("p",null,"Before start contributing, feel free to ask questions or initiate conversation via GitHub discussion.\nYou are also welcome to create issue if you encounter a bug or to suggest feature enhancements."),(0,r.kt)("p",null,"Please note we have a code of conduct, please follow it in all your interactions with the project."),(0,r.kt)("h2",{id:"best-practices"},"Best practices"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Follow the ",(0,r.kt)("a",{parentName:"li",href:"https://www.conventionalcommits.org/en/v1.0.0/"},"conventional commit")," format for all commit messages."),(0,r.kt)("li",{parentName:"ul"},"Link the PR with the issue. This is mandatory to ensure there is sufficient information for the reviewer to understand\nyour PR."),(0,r.kt)("li",{parentName:"ul"},"When you make a PR for small change (such as fixing a typo, style change, or grammar fix), please squash your commits\nso that we can maintain a cleaner git history."),(0,r.kt)("li",{parentName:"ul"},"Docs live in the code repo under ",(0,r.kt)("a",{parentName:"li",href:"https://github.com/goto/optimus/tree/main/docs"},"docs"),". Please maintain the docs\nand any docs changes can be done in the same PR."),(0,r.kt)("li",{parentName:"ul"},"Avoid force-pushing as it makes reviewing difficult.")),(0,r.kt)("h2",{id:"code-of-conduct"},"Code of Conduct"),(0,r.kt)("p",null,"Examples of behavior that contributes to creating a positive environment include:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Using welcoming and inclusive language"),(0,r.kt)("li",{parentName:"ul"},"Being respectful of differing viewpoints and experiences"),(0,r.kt)("li",{parentName:"ul"},"Gracefully accepting constructive criticism"),(0,r.kt)("li",{parentName:"ul"},"Focusing on what is best for the project")),(0,r.kt)("p",null,"Things to keep in mind before creating a new commit:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Go through the project code conventions."),(0,r.kt)("li",{parentName:"ul"},"Commit ",(0,r.kt)("a",{parentName:"li",href:"https://www.conventionalcommits.org/en/v1.0.0/"},"guidelines")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("a",{parentName:"li",href:"https://github.com/cncf/foundation/blob/master/code-of-conduct.md"},"CNCF Code of Conduct"))))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/260cf76e.d9d2c58f.js b/assets/js/260cf76e.a8e67b57.js similarity index 98% rename from assets/js/260cf76e.d9d2c58f.js rename to assets/js/260cf76e.a8e67b57.js index 519ba003b4..7bdb7b1df2 100644 --- a/assets/js/260cf76e.d9d2c58f.js +++ b/assets/js/260cf76e.a8e67b57.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3541],{3905:(e,t,r)=>{r.d(t,{Zo:()=>s,kt:()=>m});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var u=n.createContext({}),p=function(e){var t=n.useContext(u),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},s=function(e){var t=p(e.components);return n.createElement(u.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},g=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,u=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),c=p(r),g=a,m=c["".concat(u,".").concat(g)]||c[g]||d[g]||o;return r?n.createElement(m,i(i({ref:t},s),{},{components:r})):n.createElement(m,i({ref:t},s))}));function m(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=g;var l={};for(var u in t)hasOwnProperty.call(t,u)&&(l[u]=t[u]);l.originalType=e,l[c]="string"==typeof e?e:a,i[1]=l;for(var p=2;p{r.r(t),r.d(t,{assets:()=>u,contentTitle:()=>i,default:()=>d,frontMatter:()=>o,metadata:()=>l,toc:()=>p});var n=r(7462),a=(r(7294),r(3905));const o={},i="Server Configuration",l={unversionedId:"server-guide/configuration",id:"server-guide/configuration",title:"Server Configuration",description:"See the server configuration example on config.sample.yaml.",source:"@site/docs/server-guide/configuration.md",sourceDirName:"server-guide",slug:"/server-guide/configuration",permalink:"/optimus/docs/server-guide/configuration",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/server-guide/configuration.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Replay & Backup",permalink:"/optimus/docs/concepts/replay-and-backup"},next:{title:"Installing Plugins",permalink:"/optimus/docs/server-guide/installing-plugins"}},u={},p=[],s={toc:p},c="wrapper";function d(e){let{components:t,...r}=e;return(0,a.kt)(c,(0,n.Z)({},s,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"server-configuration"},"Server Configuration"),(0,a.kt)("p",null,"See the server configuration example on config.sample.yaml."),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Configuration"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Log"),(0,a.kt)("td",{parentName:"tr",align:null},"Logging level & format configuration.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Serve"),(0,a.kt)("td",{parentName:"tr",align:null},"Represents any configuration needed to start Optimus, such as port, host, DB details, and application key (for secrets encryption).")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Telemetry"),(0,a.kt)("td",{parentName:"tr",align:null},"Can be used for tracking and debugging using Jaeger.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Plugin"),(0,a.kt)("td",{parentName:"tr",align:null},"Optimus will try to look for the plugin artifacts through this configuration.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Resource Manager"),(0,a.kt)("td",{parentName:"tr",align:null},"If your server has jobs that are dependent on other jobs in another server, you can add that external Optimus server host as a resource manager.")))),(0,a.kt)("p",null,(0,a.kt)("em",{parentName:"p"},"Note:")),(0,a.kt)("p",null,"Application key can be randomly generated using: "),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"head -c 50 /dev/random | base64\n")),(0,a.kt)("p",null,"Just take the first 32 characters of the string."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3541],{3905:(e,t,r)=>{r.d(t,{Zo:()=>s,kt:()=>m});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var u=n.createContext({}),p=function(e){var t=n.useContext(u),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},s=function(e){var t=p(e.components);return n.createElement(u.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},g=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,u=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),c=p(r),g=a,m=c["".concat(u,".").concat(g)]||c[g]||d[g]||o;return r?n.createElement(m,i(i({ref:t},s),{},{components:r})):n.createElement(m,i({ref:t},s))}));function m(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=g;var l={};for(var u in t)hasOwnProperty.call(t,u)&&(l[u]=t[u]);l.originalType=e,l[c]="string"==typeof e?e:a,i[1]=l;for(var p=2;p{r.r(t),r.d(t,{assets:()=>u,contentTitle:()=>i,default:()=>d,frontMatter:()=>o,metadata:()=>l,toc:()=>p});var n=r(7462),a=(r(7294),r(3905));const o={},i="Server Configuration",l={unversionedId:"server-guide/configuration",id:"server-guide/configuration",title:"Server Configuration",description:"See the server configuration example on config.sample.yaml.",source:"@site/docs/server-guide/configuration.md",sourceDirName:"server-guide",slug:"/server-guide/configuration",permalink:"/optimus/docs/server-guide/configuration",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/server-guide/configuration.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Replay & Backup",permalink:"/optimus/docs/concepts/replay-and-backup"},next:{title:"Installing Plugins",permalink:"/optimus/docs/server-guide/installing-plugins"}},u={},p=[],s={toc:p},c="wrapper";function d(e){let{components:t,...r}=e;return(0,a.kt)(c,(0,n.Z)({},s,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"server-configuration"},"Server Configuration"),(0,a.kt)("p",null,"See the server configuration example on config.sample.yaml."),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Configuration"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Log"),(0,a.kt)("td",{parentName:"tr",align:null},"Logging level & format configuration.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Serve"),(0,a.kt)("td",{parentName:"tr",align:null},"Represents any configuration needed to start Optimus, such as port, host, DB details, and application key (for secrets encryption).")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Telemetry"),(0,a.kt)("td",{parentName:"tr",align:null},"Can be used for tracking and debugging using Jaeger.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Plugin"),(0,a.kt)("td",{parentName:"tr",align:null},"Optimus will try to look for the plugin artifacts through this configuration.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Resource Manager"),(0,a.kt)("td",{parentName:"tr",align:null},"If your server has jobs that are dependent on other jobs in another server, you can add that external Optimus server host as a resource manager.")))),(0,a.kt)("p",null,(0,a.kt)("em",{parentName:"p"},"Note:")),(0,a.kt)("p",null,"Application key can be randomly generated using: "),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"head -c 50 /dev/random | base64\n")),(0,a.kt)("p",null,"Just take the first 32 characters of the string."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/2be45fc7.dff5e95a.js b/assets/js/2be45fc7.1fc01053.js similarity index 99% rename from assets/js/2be45fc7.dff5e95a.js rename to assets/js/2be45fc7.1fc01053.js index 6b45b0f4e0..fa58ffa2ac 100644 --- a/assets/js/2be45fc7.dff5e95a.js +++ b/assets/js/2be45fc7.1fc01053.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2743],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>m});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var l=a.createContext({}),c=function(e){var t=a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},p=function(e){var t=c(e.components);return a.createElement(l.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},h=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,l=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),u=c(n),h=r,m=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?a.createElement(m,s(s({ref:t},p),{},{components:n})):a.createElement(m,s({ref:t},p))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,s=new Array(i);s[0]=h;var o={};for(var l in t)hasOwnProperty.call(t,l)&&(o[l]=t[l]);o.originalType=e,o[u]="string"==typeof e?e:r,s[1]=o;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>s,default:()=>d,frontMatter:()=>i,metadata:()=>o,toc:()=>c});var a=n(7462),r=(n(7294),n(3905));const i={},s=void 0,o={unversionedId:"rfcs/secret_management",id:"rfcs/secret_management",title:"secret_management",description:"- Feature Name: Secret Management",source:"@site/docs/rfcs/20211002_secret_management.md",sourceDirName:"rfcs",slug:"/rfcs/secret_management",permalink:"/optimus/docs/rfcs/secret_management",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20211002_secret_management.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20211002,frontMatter:{}},l={},c=[{value:"Using secrets",id:"using-secrets",level:4},{value:"Authentication & Authorization",id:"authentication--authorization",level:4},{value:"Optimus CLI",id:"optimus-cli",level:3},{value:"Create/Update",id:"createupdate",level:4},{value:"Delete",id:"delete",level:4},{value:"List",id:"list",level:4},{value:"Using secrets without Optimus",id:"using-secrets-without-optimus",level:3},{value:"Rotating Optimus Server key",id:"rotating-optimus-server-key",level:3},{value:"Migration",id:"migration",level:4}],p={toc:c},u="wrapper";function d(e){let{components:t,...n}=e;return(0,r.kt)(u,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Feature Name: Secret Management"),(0,r.kt)("li",{parentName:"ul"},"Status: Approved"),(0,r.kt)("li",{parentName:"ul"},"Start Date: 2021-10-02"),(0,r.kt)("li",{parentName:"ul"},"Authors: Kush Sharma & Sravan ")),(0,r.kt)("h1",{id:"summary"},"Summary"),(0,r.kt)("p",null,"A lot of transformation operations require credentials to execute, there is a need to have a convenient way to save secrets and then access them in containers during the execution. This secret may also be needed in plugin adapters to compute dependencies/compile assets/etc before the actual transformation even begin. This is currently done using registering a secret to optimus so that it can be accessed by plugins and Kubernetes opaque secret, a single secret per plugin, getting mounted in the container(i.e. not at individual job level)."),(0,r.kt)("p",null,"This can be solved by allowing users to register secret from Optimus CLI as a key value pair, storing them encrypted using a single key across all tenants."),(0,r.kt)("h1",{id:"technical-design"},"Technical Design"),(0,r.kt)("p",null,"To keep string literals as secret, it is a requirement Optimus keep them encrypted in database. Optimus Server Key is used to encrypt & decrypt the secret & will ensure the secret is encrypted at rest. Each secret is a key value pair where key is an alpha numeric literal and value is base64 encoded string. "),(0,r.kt)("p",null,"Optimus has two sets of secrets, user managed secrets & others which are needed for server operations. Each of server managed secrets should be prefixed by ",(0,r.kt)("inlineCode",{parentName:"p"},"_OPTIMUS_")," and will not be allowed to be used by users in the job spec. Optimus should also disallow anyone using this prefix to register their secrets. The secrets can be namespaced by optimus namespace or at project level, and will be accessible accordingly. Secret names should be maintained unique across a project."),(0,r.kt)("p",null,"All secret names within Optimus systems are considered case insensitive, treating different letter cases as identical. Furthermore, to promote consistency and ease of management, all secret names are uniformly stored in uppercase letters, regardless of their original format. The implementation of this policy is essential to enhance security, minimize potential inconsistencies, and facilitate seamless secret retrieval and utilization."),(0,r.kt)("h4",{id:"using-secrets"},"Using secrets"),(0,r.kt)("p",null,"Secrets can be used as part of the job spec config using macros with their names. This will work as aliasing the secret to be used in containers. Only the secrets created at project & namespace the job belongs to can be referenced. So, for the plugin writers any secret that plugin needs can be accessed through environment variables defined in the job spec or can get the secrets by defining in any assets."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},"task: foo\nconfig:\n do: this\n dsn: {{ .secret.postgres_dsn }}\n")),(0,r.kt)("p",null,"One thing to note is currently we print all the container environment variables using ",(0,r.kt)("inlineCode",{parentName:"p"},"printenv")," command as debug. This should be removed after this RFC is merged to avoid exposing secrets in container logs."),(0,r.kt)("p",null,"Only the admins & containers to be authorized for ",(0,r.kt)("inlineCode",{parentName:"p"},"registerinstance")," end point, as this will allow access to all secrets."),(0,r.kt)("p",null,"Because Optimus is deployed in trusted network, we don't need TLS for now to fetch job secrets but once Optimus is deployed as a service on edge network, this communication should only happen over TLS. "),(0,r.kt)("h4",{id:"authentication--authorization"},"Authentication & Authorization"),(0,r.kt)("p",null,"Even though Optimus doesn't have its own authentication, expect users to bring in their own auth proxy infront of Optimus. All user access & container access will be restricted through the auth proxy. The corresponding secret through which the containers running in the kubernetes cluster will be authenticated need to be precreated per project."),(0,r.kt)("h3",{id:"optimus-cli"},"Optimus CLI"),(0,r.kt)("p",null,"User interaction to manage a secret will start from CLI. Users can create/update/list/delete a secret as follows"),(0,r.kt)("p",null,"By default secrets will be created under their namespace, but optionally the secret can be created at project level by not providing any namespace while creation. This is needed if users want to allow access across entire project."),(0,r.kt)("p",null,"Secrets can be accessed by providing the project & namespace the secret is created in, if the secret is created at project level then namespace can be set to empty string if optimus.yaml already has the namespace configured."),(0,r.kt)("h4",{id:"createupdate"},"Create/Update"),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"optimus secret create/update ")," will take a secret name and value"),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},'optimus secret create/update --file="path"')," should read the file content as value. "),(0,r.kt)("p",null,"Additional flag ",(0,r.kt)("inlineCode",{parentName:"p"},"--base64")," can be provided by user stating the value is already encoded, if not provided optimus ensures to encode & store it, basic checks can be done to check if the string is a valid base64 encoded string."),(0,r.kt)("h4",{id:"delete"},"Delete"),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"optimus secret delete ")," "),(0,r.kt)("h4",{id:"list"},"List"),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"optimus secret list")," to list all created secrets in a project/namespace, along with the creation/updated time, will be helpful such that users can use in the job spec, as users might forget the key name, this will not list the system managed secrets."),(0,r.kt)("p",null,"List operation will print a digest of the secret. Digest should be a SHA hash of the encrypted string to simply visualize it as a signature when a secret is changed or the key gets rotated."),(0,r.kt)("p",null," Example:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"}," NAME | DIGEST | NAMESPACE | DATE\n SECRET_1 | 6c463e806738046ff3c78a08d8bd2b70 | * | 2021-10-06 02:02:02\n SECRET_2 | 3aa788a21a76651c349ceeee76f1cb76 | finance | 2021-10-06 06:02:02\n SECRET_2 | 3aa788a21a76651c349ceeee76f1cb76 | transport | 2021-10-06 06:02:02\n")),(0,r.kt)("p",null,"This command will only shows the user managed secret sets and ignoring the system managed secret, while on the REST response\nboth sets can be shown. An additional field in secret table called 'TYPE' can be added to differentiate the two sets. "),(0,r.kt)("h3",{id:"using-secrets-without-optimus"},"Using secrets without Optimus"),(0,r.kt)("p",null,"If someone wants to pass an exclusive secret without registering it with Optimus first, that should also be possible. "),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"In case of k8s: this can be done using a new field introduced in Job spec as ",(0,r.kt)("inlineCode",{parentName:"li"},"metadata")," which will allow users to mount arbitrary secrets inside the container available in the same k8s namespace.")),(0,r.kt)("h3",{id:"rotating-optimus-server-key"},"Rotating Optimus Server key"),(0,r.kt)("p",null,"There is a need for rotating Optimus Server Key when it is compromised. As the server key is configured through environment variable, the rotation can happen by configuring through environment variables. There can be two environment variables for server keys ",(0,r.kt)("inlineCode",{parentName:"p"},"OLD_APP_KEY")," & ",(0,r.kt)("inlineCode",{parentName:"p"},"APP_KEY"),". During startup sha of the ",(0,r.kt)("inlineCode",{parentName:"p"},"OLD_APP_KEY")," is compared with the sha stored in the database, if it matches then rotation will happen and at the end of rotation the sha will be replaced with ",(0,r.kt)("inlineCode",{parentName:"p"},"APP_KEY's")," sha. The comparision is needed to check to not attempt rotation during restarts. If there are multiple replicas then as we do this in a transaction only one succeeds."),(0,r.kt)("p",null,"This step will internally loading all the secrets that belong to a project to memory, decrypting it with the old_key, and encrypting it with the new key, the entire operation will happen in a single db transaction."),(0,r.kt)("h4",{id:"migration"},"Migration"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"This design will be a breaking change compare to how the secrets are handled and will require all the current secrets to be registered again.\nCurrent system managed secrets will be re-registered using ",(0,r.kt)("inlineCode",{parentName:"li"},"_OPTIMUS_")," prefix. Plugin secrets will also need to be registered to Optimus.")),(0,r.kt)("h1",{id:"footnotes--references"},"Footnotes & References"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Multi party encryption via ",(0,r.kt)("a",{parentName:"li",href:"https://github.com/FiloSottile/age"},"age")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("a",{parentName:"li",href:"https://gocloud.dev/howto/secrets/"},"Key Management Services "))))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2743],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>m});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function s(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var l=a.createContext({}),c=function(e){var t=a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},p=function(e){var t=c(e.components);return a.createElement(l.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},h=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,l=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),u=c(n),h=r,m=u["".concat(l,".").concat(h)]||u[h]||d[h]||i;return n?a.createElement(m,s(s({ref:t},p),{},{components:n})):a.createElement(m,s({ref:t},p))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,s=new Array(i);s[0]=h;var o={};for(var l in t)hasOwnProperty.call(t,l)&&(o[l]=t[l]);o.originalType=e,o[u]="string"==typeof e?e:r,s[1]=o;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>s,default:()=>d,frontMatter:()=>i,metadata:()=>o,toc:()=>c});var a=n(7462),r=(n(7294),n(3905));const i={},s=void 0,o={unversionedId:"rfcs/secret_management",id:"rfcs/secret_management",title:"secret_management",description:"- Feature Name: Secret Management",source:"@site/docs/rfcs/20211002_secret_management.md",sourceDirName:"rfcs",slug:"/rfcs/secret_management",permalink:"/optimus/docs/rfcs/secret_management",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20211002_secret_management.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20211002,frontMatter:{}},l={},c=[{value:"Using secrets",id:"using-secrets",level:4},{value:"Authentication & Authorization",id:"authentication--authorization",level:4},{value:"Optimus CLI",id:"optimus-cli",level:3},{value:"Create/Update",id:"createupdate",level:4},{value:"Delete",id:"delete",level:4},{value:"List",id:"list",level:4},{value:"Using secrets without Optimus",id:"using-secrets-without-optimus",level:3},{value:"Rotating Optimus Server key",id:"rotating-optimus-server-key",level:3},{value:"Migration",id:"migration",level:4}],p={toc:c},u="wrapper";function d(e){let{components:t,...n}=e;return(0,r.kt)(u,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Feature Name: Secret Management"),(0,r.kt)("li",{parentName:"ul"},"Status: Approved"),(0,r.kt)("li",{parentName:"ul"},"Start Date: 2021-10-02"),(0,r.kt)("li",{parentName:"ul"},"Authors: Kush Sharma & Sravan ")),(0,r.kt)("h1",{id:"summary"},"Summary"),(0,r.kt)("p",null,"A lot of transformation operations require credentials to execute, there is a need to have a convenient way to save secrets and then access them in containers during the execution. This secret may also be needed in plugin adapters to compute dependencies/compile assets/etc before the actual transformation even begin. This is currently done using registering a secret to optimus so that it can be accessed by plugins and Kubernetes opaque secret, a single secret per plugin, getting mounted in the container(i.e. not at individual job level)."),(0,r.kt)("p",null,"This can be solved by allowing users to register secret from Optimus CLI as a key value pair, storing them encrypted using a single key across all tenants."),(0,r.kt)("h1",{id:"technical-design"},"Technical Design"),(0,r.kt)("p",null,"To keep string literals as secret, it is a requirement Optimus keep them encrypted in database. Optimus Server Key is used to encrypt & decrypt the secret & will ensure the secret is encrypted at rest. Each secret is a key value pair where key is an alpha numeric literal and value is base64 encoded string. "),(0,r.kt)("p",null,"Optimus has two sets of secrets, user managed secrets & others which are needed for server operations. Each of server managed secrets should be prefixed by ",(0,r.kt)("inlineCode",{parentName:"p"},"_OPTIMUS_")," and will not be allowed to be used by users in the job spec. Optimus should also disallow anyone using this prefix to register their secrets. The secrets can be namespaced by optimus namespace or at project level, and will be accessible accordingly. Secret names should be maintained unique across a project."),(0,r.kt)("p",null,"All secret names within Optimus systems are considered case insensitive, treating different letter cases as identical. Furthermore, to promote consistency and ease of management, all secret names are uniformly stored in uppercase letters, regardless of their original format. The implementation of this policy is essential to enhance security, minimize potential inconsistencies, and facilitate seamless secret retrieval and utilization."),(0,r.kt)("h4",{id:"using-secrets"},"Using secrets"),(0,r.kt)("p",null,"Secrets can be used as part of the job spec config using macros with their names. This will work as aliasing the secret to be used in containers. Only the secrets created at project & namespace the job belongs to can be referenced. So, for the plugin writers any secret that plugin needs can be accessed through environment variables defined in the job spec or can get the secrets by defining in any assets."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},"task: foo\nconfig:\n do: this\n dsn: {{ .secret.postgres_dsn }}\n")),(0,r.kt)("p",null,"One thing to note is currently we print all the container environment variables using ",(0,r.kt)("inlineCode",{parentName:"p"},"printenv")," command as debug. This should be removed after this RFC is merged to avoid exposing secrets in container logs."),(0,r.kt)("p",null,"Only the admins & containers to be authorized for ",(0,r.kt)("inlineCode",{parentName:"p"},"registerinstance")," end point, as this will allow access to all secrets."),(0,r.kt)("p",null,"Because Optimus is deployed in trusted network, we don't need TLS for now to fetch job secrets but once Optimus is deployed as a service on edge network, this communication should only happen over TLS. "),(0,r.kt)("h4",{id:"authentication--authorization"},"Authentication & Authorization"),(0,r.kt)("p",null,"Even though Optimus doesn't have its own authentication, expect users to bring in their own auth proxy infront of Optimus. All user access & container access will be restricted through the auth proxy. The corresponding secret through which the containers running in the kubernetes cluster will be authenticated need to be precreated per project."),(0,r.kt)("h3",{id:"optimus-cli"},"Optimus CLI"),(0,r.kt)("p",null,"User interaction to manage a secret will start from CLI. Users can create/update/list/delete a secret as follows"),(0,r.kt)("p",null,"By default secrets will be created under their namespace, but optionally the secret can be created at project level by not providing any namespace while creation. This is needed if users want to allow access across entire project."),(0,r.kt)("p",null,"Secrets can be accessed by providing the project & namespace the secret is created in, if the secret is created at project level then namespace can be set to empty string if optimus.yaml already has the namespace configured."),(0,r.kt)("h4",{id:"createupdate"},"Create/Update"),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"optimus secret create/update ")," will take a secret name and value"),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},'optimus secret create/update --file="path"')," should read the file content as value. "),(0,r.kt)("p",null,"Additional flag ",(0,r.kt)("inlineCode",{parentName:"p"},"--base64")," can be provided by user stating the value is already encoded, if not provided optimus ensures to encode & store it, basic checks can be done to check if the string is a valid base64 encoded string."),(0,r.kt)("h4",{id:"delete"},"Delete"),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"optimus secret delete ")," "),(0,r.kt)("h4",{id:"list"},"List"),(0,r.kt)("p",null,(0,r.kt)("inlineCode",{parentName:"p"},"optimus secret list")," to list all created secrets in a project/namespace, along with the creation/updated time, will be helpful such that users can use in the job spec, as users might forget the key name, this will not list the system managed secrets."),(0,r.kt)("p",null,"List operation will print a digest of the secret. Digest should be a SHA hash of the encrypted string to simply visualize it as a signature when a secret is changed or the key gets rotated."),(0,r.kt)("p",null," Example:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"}," NAME | DIGEST | NAMESPACE | DATE\n SECRET_1 | 6c463e806738046ff3c78a08d8bd2b70 | * | 2021-10-06 02:02:02\n SECRET_2 | 3aa788a21a76651c349ceeee76f1cb76 | finance | 2021-10-06 06:02:02\n SECRET_2 | 3aa788a21a76651c349ceeee76f1cb76 | transport | 2021-10-06 06:02:02\n")),(0,r.kt)("p",null,"This command will only shows the user managed secret sets and ignoring the system managed secret, while on the REST response\nboth sets can be shown. An additional field in secret table called 'TYPE' can be added to differentiate the two sets. "),(0,r.kt)("h3",{id:"using-secrets-without-optimus"},"Using secrets without Optimus"),(0,r.kt)("p",null,"If someone wants to pass an exclusive secret without registering it with Optimus first, that should also be possible. "),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"In case of k8s: this can be done using a new field introduced in Job spec as ",(0,r.kt)("inlineCode",{parentName:"li"},"metadata")," which will allow users to mount arbitrary secrets inside the container available in the same k8s namespace.")),(0,r.kt)("h3",{id:"rotating-optimus-server-key"},"Rotating Optimus Server key"),(0,r.kt)("p",null,"There is a need for rotating Optimus Server Key when it is compromised. As the server key is configured through environment variable, the rotation can happen by configuring through environment variables. There can be two environment variables for server keys ",(0,r.kt)("inlineCode",{parentName:"p"},"OLD_APP_KEY")," & ",(0,r.kt)("inlineCode",{parentName:"p"},"APP_KEY"),". During startup sha of the ",(0,r.kt)("inlineCode",{parentName:"p"},"OLD_APP_KEY")," is compared with the sha stored in the database, if it matches then rotation will happen and at the end of rotation the sha will be replaced with ",(0,r.kt)("inlineCode",{parentName:"p"},"APP_KEY's")," sha. The comparision is needed to check to not attempt rotation during restarts. If there are multiple replicas then as we do this in a transaction only one succeeds."),(0,r.kt)("p",null,"This step will internally loading all the secrets that belong to a project to memory, decrypting it with the old_key, and encrypting it with the new key, the entire operation will happen in a single db transaction."),(0,r.kt)("h4",{id:"migration"},"Migration"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"This design will be a breaking change compare to how the secrets are handled and will require all the current secrets to be registered again.\nCurrent system managed secrets will be re-registered using ",(0,r.kt)("inlineCode",{parentName:"li"},"_OPTIMUS_")," prefix. Plugin secrets will also need to be registered to Optimus.")),(0,r.kt)("h1",{id:"footnotes--references"},"Footnotes & References"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Multi party encryption via ",(0,r.kt)("a",{parentName:"li",href:"https://github.com/FiloSottile/age"},"age")),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("a",{parentName:"li",href:"https://gocloud.dev/howto/secrets/"},"Key Management Services "))))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/3278defe.4c2877e3.js b/assets/js/3278defe.98e4e90e.js similarity index 98% rename from assets/js/3278defe.4c2877e3.js rename to assets/js/3278defe.98e4e90e.js index e048b90008..d30180673d 100644 --- a/assets/js/3278defe.4c2877e3.js +++ b/assets/js/3278defe.98e4e90e.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[1675],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>m});var r=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var a=r.createContext({}),l=function(e){var t=r.useContext(a),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return r.createElement(a.Provider,{value:t},e.children)},p="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},f=r.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,a=e.parentName,u=c(e,["components","mdxType","originalType","parentName"]),p=l(n),f=i,m=p["".concat(a,".").concat(f)]||p[f]||d[f]||o;return n?r.createElement(m,s(s({ref:t},u),{},{components:n})):r.createElement(m,s({ref:t},u))}));function m(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=f;var c={};for(var a in t)hasOwnProperty.call(t,a)&&(c[a]=t[a]);c.originalType=e,c[p]="string"==typeof e?e:i,s[1]=c;for(var l=2;l{n.r(t),n.d(t,{assets:()=>a,contentTitle:()=>s,default:()=>d,frontMatter:()=>o,metadata:()=>c,toc:()=>l});var r=n(7462),i=(n(7294),n(3905));const o={},s="Defining Scheduler Version",c={unversionedId:"client-guide/defining-scheduler-version",id:"client-guide/defining-scheduler-version",title:"Defining Scheduler Version",description:"For now, optimus only supports airflow as a scheduler. Optimus provides capability to determine the scheduler version per project by defining scheduler_version in project config (optimus.yaml). By default, optimus use airflow version 2.1 if it is not specified in optimus.yaml config.",source:"@site/docs/client-guide/defining-scheduler-version.md",sourceDirName:"client-guide",slug:"/client-guide/defining-scheduler-version",permalink:"/optimus/docs/client-guide/defining-scheduler-version",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/defining-scheduler-version.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Work with Extension",permalink:"/optimus/docs/client-guide/work-with-extension"},next:{title:"Introduction of Plugin Development",permalink:"/optimus/docs/building-plugin/introduction"}},a={},l=[],u={toc:l},p="wrapper";function d(e){let{components:t,...n}=e;return(0,i.kt)(p,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"defining-scheduler-version"},"Defining Scheduler Version"),(0,i.kt)("p",null,"For now, optimus only supports airflow as a scheduler. Optimus provides capability to determine the scheduler version per project by defining ",(0,i.kt)("inlineCode",{parentName:"p"},"scheduler_version")," in project config (",(0,i.kt)("inlineCode",{parentName:"p"},"optimus.yaml"),"). By default, optimus use airflow version 2.1 if it is not specified in ",(0,i.kt)("inlineCode",{parentName:"p"},"optimus.yaml")," config."),(0,i.kt)("p",null,"Optimus supports these following version:\n| Version |\n|---|\n| 2.1 |\n| 2.4 |"))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[1675],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>m});var r=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var a=r.createContext({}),l=function(e){var t=r.useContext(a),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},u=function(e){var t=l(e.components);return r.createElement(a.Provider,{value:t},e.children)},p="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},f=r.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,a=e.parentName,u=c(e,["components","mdxType","originalType","parentName"]),p=l(n),f=i,m=p["".concat(a,".").concat(f)]||p[f]||d[f]||o;return n?r.createElement(m,s(s({ref:t},u),{},{components:n})):r.createElement(m,s({ref:t},u))}));function m(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=f;var c={};for(var a in t)hasOwnProperty.call(t,a)&&(c[a]=t[a]);c.originalType=e,c[p]="string"==typeof e?e:i,s[1]=c;for(var l=2;l{n.r(t),n.d(t,{assets:()=>a,contentTitle:()=>s,default:()=>d,frontMatter:()=>o,metadata:()=>c,toc:()=>l});var r=n(7462),i=(n(7294),n(3905));const o={},s="Defining Scheduler Version",c={unversionedId:"client-guide/defining-scheduler-version",id:"client-guide/defining-scheduler-version",title:"Defining Scheduler Version",description:"For now, optimus only supports airflow as a scheduler. Optimus provides capability to determine the scheduler version per project by defining scheduler_version in project config (optimus.yaml). By default, optimus use airflow version 2.1 if it is not specified in optimus.yaml config.",source:"@site/docs/client-guide/defining-scheduler-version.md",sourceDirName:"client-guide",slug:"/client-guide/defining-scheduler-version",permalink:"/optimus/docs/client-guide/defining-scheduler-version",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/defining-scheduler-version.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Work with Extension",permalink:"/optimus/docs/client-guide/work-with-extension"},next:{title:"Introduction of Plugin Development",permalink:"/optimus/docs/building-plugin/introduction"}},a={},l=[],u={toc:l},p="wrapper";function d(e){let{components:t,...n}=e;return(0,i.kt)(p,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"defining-scheduler-version"},"Defining Scheduler Version"),(0,i.kt)("p",null,"For now, optimus only supports airflow as a scheduler. Optimus provides capability to determine the scheduler version per project by defining ",(0,i.kt)("inlineCode",{parentName:"p"},"scheduler_version")," in project config (",(0,i.kt)("inlineCode",{parentName:"p"},"optimus.yaml"),"). By default, optimus use airflow version 2.1 if it is not specified in ",(0,i.kt)("inlineCode",{parentName:"p"},"optimus.yaml")," config."),(0,i.kt)("p",null,"Optimus supports these following version:\n| Version |\n|---|\n| 2.1 |\n| 2.4 |"))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/420f7ef9.825be566.js b/assets/js/420f7ef9.16f54e78.js similarity index 98% rename from assets/js/420f7ef9.825be566.js rename to assets/js/420f7ef9.16f54e78.js index 56efd4d437..89a658330d 100644 --- a/assets/js/420f7ef9.825be566.js +++ b/assets/js/420f7ef9.16f54e78.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3220],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>f});var o=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function a(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var l=o.createContext({}),c=function(e){var t=o.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},p=function(e){var t=c(e.components);return o.createElement(l.Provider,{value:t},e.children)},d="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},m=o.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),m=r,f=d["".concat(l,".").concat(m)]||d[m]||u[m]||i;return n?o.createElement(f,a(a({ref:t},p),{},{components:n})):o.createElement(f,a({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=m;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[d]="string"==typeof e?e:r,a[1]=s;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>a,default:()=>u,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var o=n(7462),r=(n(7294),n(3905));const i={},a="Applying Job Specifications",s={unversionedId:"client-guide/applying-job-specifications",id:"client-guide/applying-job-specifications",title:"Applying Job Specifications",description:"Once you have the job specifications ready, let\u2019s try to deploy the jobs to the server by running this command:",source:"@site/docs/client-guide/applying-job-specifications.md",sourceDirName:"client-guide",slug:"/client-guide/applying-job-specifications",permalink:"/optimus/docs/client-guide/applying-job-specifications",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/applying-job-specifications.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Verifying the Jobs",permalink:"/optimus/docs/client-guide/verifying-jobs"},next:{title:"Uploading Job to Scheduler",permalink:"/optimus/docs/client-guide/uploading-jobs-to-scheduler"}},l={},c=[],p={toc:c},d="wrapper";function u(e){let{components:t,...n}=e;return(0,r.kt)(d,(0,o.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"applying-job-specifications"},"Applying Job Specifications"),(0,r.kt)("p",null,"Once you have the job specifications ready, let\u2019s try to deploy the jobs to the server by running this command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job replace-all --verbose\n")),(0,r.kt)("p",null,"Note: add --config flag if you are not in the same directory with your client configuration (optimus.yaml)."),(0,r.kt)("p",null,"This replace-all command works per project or namespace level and will try to compare the incoming jobs and the jobs\nin the server. You will find in the logs how many jobs are new, modified, and deleted based on the current condition."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job replace-all --verbose\n\n> Validating namespaces\nvalidation finished!\n\n> Replacing all jobs for namespaces [sample_namespace]\n> Receiving responses:\n[sample_namespace] received 1 job specs\n[sample_namespace] found 1 new, 0 modified, and 0 deleted job specs\n[sample_namespace] processing job job1\n[sample_namespace] successfully added 1 jobs\nreplace all job specifications finished!\n")),(0,r.kt)("p",null,"You might notice based on the log that Optimus tries to find which jobs are new, modified, or deleted. This is because\nOptimus will not try to process every job in every single ",(0,r.kt)("inlineCode",{parentName:"p"},"replace-all")," command for performance reasons. If you have\nneeds to refresh all of the jobs in the project from the server, regardless it has changed or not, do run the below command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job refresh --verbose\n")),(0,r.kt)("p",null,"This refresh command is not taking any specifications as a request. It will only refresh the jobs in the server."),(0,r.kt)("p",null,"Also, do notice that these ",(0,r.kt)("strong",{parentName:"p"},"replace-all")," and ",(0,r.kt)("strong",{parentName:"p"},"refresh")," commands are only for registering the job specifications in the server,\nincluding resolving the dependencies. After this, you can compile and upload the jobs to the scheduler using the\n",(0,r.kt)("inlineCode",{parentName:"p"},"scheduler upload-all")," ",(0,r.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/uploading-jobs-to-scheduler"},"command"),"."),(0,r.kt)("p",null,"Note: Currently Optimus does not provide a way to deploy only a single job through CLI. This capability is being\nsupported in the API."))}u.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3220],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>f});var o=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function a(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var l=o.createContext({}),c=function(e){var t=o.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},p=function(e){var t=c(e.components);return o.createElement(l.Provider,{value:t},e.children)},d="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},m=o.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),m=r,f=d["".concat(l,".").concat(m)]||d[m]||u[m]||i;return n?o.createElement(f,a(a({ref:t},p),{},{components:n})):o.createElement(f,a({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,a=new Array(i);a[0]=m;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[d]="string"==typeof e?e:r,a[1]=s;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>a,default:()=>u,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var o=n(7462),r=(n(7294),n(3905));const i={},a="Applying Job Specifications",s={unversionedId:"client-guide/applying-job-specifications",id:"client-guide/applying-job-specifications",title:"Applying Job Specifications",description:"Once you have the job specifications ready, let\u2019s try to deploy the jobs to the server by running this command:",source:"@site/docs/client-guide/applying-job-specifications.md",sourceDirName:"client-guide",slug:"/client-guide/applying-job-specifications",permalink:"/optimus/docs/client-guide/applying-job-specifications",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/applying-job-specifications.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Verifying the Jobs",permalink:"/optimus/docs/client-guide/verifying-jobs"},next:{title:"Uploading Job to Scheduler",permalink:"/optimus/docs/client-guide/uploading-jobs-to-scheduler"}},l={},c=[],p={toc:c},d="wrapper";function u(e){let{components:t,...n}=e;return(0,r.kt)(d,(0,o.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"applying-job-specifications"},"Applying Job Specifications"),(0,r.kt)("p",null,"Once you have the job specifications ready, let\u2019s try to deploy the jobs to the server by running this command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job replace-all --verbose\n")),(0,r.kt)("p",null,"Note: add --config flag if you are not in the same directory with your client configuration (optimus.yaml)."),(0,r.kt)("p",null,"This replace-all command works per project or namespace level and will try to compare the incoming jobs and the jobs\nin the server. You will find in the logs how many jobs are new, modified, and deleted based on the current condition."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job replace-all --verbose\n\n> Validating namespaces\nvalidation finished!\n\n> Replacing all jobs for namespaces [sample_namespace]\n> Receiving responses:\n[sample_namespace] received 1 job specs\n[sample_namespace] found 1 new, 0 modified, and 0 deleted job specs\n[sample_namespace] processing job job1\n[sample_namespace] successfully added 1 jobs\nreplace all job specifications finished!\n")),(0,r.kt)("p",null,"You might notice based on the log that Optimus tries to find which jobs are new, modified, or deleted. This is because\nOptimus will not try to process every job in every single ",(0,r.kt)("inlineCode",{parentName:"p"},"replace-all")," command for performance reasons. If you have\nneeds to refresh all of the jobs in the project from the server, regardless it has changed or not, do run the below command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job refresh --verbose\n")),(0,r.kt)("p",null,"This refresh command is not taking any specifications as a request. It will only refresh the jobs in the server."),(0,r.kt)("p",null,"Also, do notice that these ",(0,r.kt)("strong",{parentName:"p"},"replace-all")," and ",(0,r.kt)("strong",{parentName:"p"},"refresh")," commands are only for registering the job specifications in the server,\nincluding resolving the dependencies. After this, you can compile and upload the jobs to the scheduler using the\n",(0,r.kt)("inlineCode",{parentName:"p"},"scheduler upload-all")," ",(0,r.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/uploading-jobs-to-scheduler"},"command"),"."),(0,r.kt)("p",null,"Note: Currently Optimus does not provide a way to deploy only a single job through CLI. This capability is being\nsupported in the API."))}u.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/4f8cfcf0.0e6f5ff6.js b/assets/js/4f8cfcf0.40f2e70e.js similarity index 98% rename from assets/js/4f8cfcf0.0e6f5ff6.js rename to assets/js/4f8cfcf0.40f2e70e.js index 26f6416047..7e45b83ab4 100644 --- a/assets/js/4f8cfcf0.0e6f5ff6.js +++ b/assets/js/4f8cfcf0.40f2e70e.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8210],{3905:(e,t,n)=>{n.d(t,{Zo:()=>s,kt:()=>f});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function a(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=r.createContext({}),p=function(e){var t=r.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},s=function(e){var t=p(e.components);return r.createElement(c.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,c=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=p(n),m=o,f=u["".concat(c,".").concat(m)]||u[m]||d[m]||i;return n?r.createElement(f,a(a({ref:t},s),{},{components:n})):r.createElement(f,a({ref:t},s))}));function f(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=m;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l[u]="string"==typeof e?e:o,a[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>a,default:()=>d,frontMatter:()=>i,metadata:()=>l,toc:()=>p});var r=n(7462),o=(n(7294),n(3905));const i={},a="Uploading Job to Scheduler",l={unversionedId:"client-guide/uploading-jobs-to-scheduler",id:"client-guide/uploading-jobs-to-scheduler",title:"Uploading Job to Scheduler",description:"Compile and upload all jobs in the project by using this command:",source:"@site/docs/client-guide/uploading-jobs-to-scheduler.md",sourceDirName:"client-guide",slug:"/client-guide/uploading-jobs-to-scheduler",permalink:"/optimus/docs/client-guide/uploading-jobs-to-scheduler",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/uploading-jobs-to-scheduler.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Applying Job Specifications",permalink:"/optimus/docs/client-guide/applying-job-specifications"},next:{title:"Organizing Specifications",permalink:"/optimus/docs/client-guide/organizing-specifications"}},c={},p=[],s={toc:p},u="wrapper";function d(e){let{components:t,...n}=e;return(0,o.kt)(u,(0,r.Z)({},s,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"uploading-job-to-scheduler"},"Uploading Job to Scheduler"),(0,o.kt)("p",null,"Compile and upload all jobs in the project by using this command:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus scheduler upload-all\n")),(0,o.kt)("p",null,(0,o.kt)("em",{parentName:"p"},"Note: add --config flag if you are not in the same directory with your client configuration (optimus.yaml).")),(0,o.kt)("p",null,"This command will compile all of the jobs in the project to Airflow DAG files and will store the result to the path\nthat has been set as ",(0,o.kt)("inlineCode",{parentName:"p"},"STORAGE_PATH")," in the project configuration. Do note that ",(0,o.kt)("inlineCode",{parentName:"p"},"STORAGE")," secret might be needed if\nthe storage requires a credential."),(0,o.kt)("p",null,"Once you have the DAG files in the storage, you can sync the files to Airflow as you\u2019d like."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8210],{3905:(e,t,n)=>{n.d(t,{Zo:()=>s,kt:()=>f});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function a(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=r.createContext({}),p=function(e){var t=r.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},s=function(e){var t=p(e.components);return r.createElement(c.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,c=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),u=p(n),m=o,f=u["".concat(c,".").concat(m)]||u[m]||d[m]||i;return n?r.createElement(f,a(a({ref:t},s),{},{components:n})):r.createElement(f,a({ref:t},s))}));function f(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=m;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l[u]="string"==typeof e?e:o,a[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>a,default:()=>d,frontMatter:()=>i,metadata:()=>l,toc:()=>p});var r=n(7462),o=(n(7294),n(3905));const i={},a="Uploading Job to Scheduler",l={unversionedId:"client-guide/uploading-jobs-to-scheduler",id:"client-guide/uploading-jobs-to-scheduler",title:"Uploading Job to Scheduler",description:"Compile and upload all jobs in the project by using this command:",source:"@site/docs/client-guide/uploading-jobs-to-scheduler.md",sourceDirName:"client-guide",slug:"/client-guide/uploading-jobs-to-scheduler",permalink:"/optimus/docs/client-guide/uploading-jobs-to-scheduler",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/uploading-jobs-to-scheduler.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Applying Job Specifications",permalink:"/optimus/docs/client-guide/applying-job-specifications"},next:{title:"Organizing Specifications",permalink:"/optimus/docs/client-guide/organizing-specifications"}},c={},p=[],s={toc:p},u="wrapper";function d(e){let{components:t,...n}=e;return(0,o.kt)(u,(0,r.Z)({},s,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"uploading-job-to-scheduler"},"Uploading Job to Scheduler"),(0,o.kt)("p",null,"Compile and upload all jobs in the project by using this command:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus scheduler upload-all\n")),(0,o.kt)("p",null,(0,o.kt)("em",{parentName:"p"},"Note: add --config flag if you are not in the same directory with your client configuration (optimus.yaml).")),(0,o.kt)("p",null,"This command will compile all of the jobs in the project to Airflow DAG files and will store the result to the path\nthat has been set as ",(0,o.kt)("inlineCode",{parentName:"p"},"STORAGE_PATH")," in the project configuration. Do note that ",(0,o.kt)("inlineCode",{parentName:"p"},"STORAGE")," secret might be needed if\nthe storage requires a credential."),(0,o.kt)("p",null,"Once you have the DAG files in the storage, you can sync the files to Airflow as you\u2019d like."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/537b8c88.2571e53c.js b/assets/js/537b8c88.e93484a0.js similarity index 98% rename from assets/js/537b8c88.2571e53c.js rename to assets/js/537b8c88.e93484a0.js index 33c47dec00..236acf4532 100644 --- a/assets/js/537b8c88.2571e53c.js +++ b/assets/js/537b8c88.e93484a0.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2369],{3905:(e,t,r)=>{r.d(t,{Zo:()=>s,kt:()=>f});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function a(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var c=n.createContext({}),u=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):a(a({},t),e)),r},s=function(e){var t=u(e.components);return n.createElement(c.Provider,{value:t},e.children)},l="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,i=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),l=u(r),m=o,f=l["".concat(c,".").concat(m)]||l[m]||d[m]||i;return r?n.createElement(f,a(a({ref:t},s),{},{components:r})):n.createElement(f,a({ref:t},s))}));function f(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=r.length,a=new Array(i);a[0]=m;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p[l]="string"==typeof e?e:o,a[1]=p;for(var u=2;u{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>a,default:()=>d,frontMatter:()=>i,metadata:()=>p,toc:()=>u});var n=r(7462),o=(r(7294),r(3905));const i={},a="Developer Environment Setup",p={unversionedId:"contribute/developer-env-setup",id:"contribute/developer-env-setup",title:"Developer Environment Setup",description:"To test Optimus including the integration with Airflow, as well as testing the runtime interactions, Optimus provides",source:"@site/docs/contribute/developer-env-setup.md",sourceDirName:"contribute",slug:"/contribute/developer-env-setup",permalink:"/optimus/docs/contribute/developer-env-setup",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/contribute/developer-env-setup.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Contributing",permalink:"/optimus/docs/contribute/contribution-process"},next:{title:"API",permalink:"/optimus/docs/reference/api"}},c={},u=[],s={toc:u},l="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(l,(0,n.Z)({},s,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"developer-environment-setup"},"Developer Environment Setup"),(0,o.kt)("p",null,"To test Optimus including the integration with Airflow, as well as testing the runtime interactions, Optimus provides\na way to simplify setting up the environment. Take a look and follow the guide in dev setup ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/optimus/tree/main/dev"},"section"),"."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2369],{3905:(e,t,r)=>{r.d(t,{Zo:()=>s,kt:()=>f});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function a(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var c=n.createContext({}),u=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):a(a({},t),e)),r},s=function(e){var t=u(e.components);return n.createElement(c.Provider,{value:t},e.children)},l="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,i=e.originalType,c=e.parentName,s=p(e,["components","mdxType","originalType","parentName"]),l=u(r),m=o,f=l["".concat(c,".").concat(m)]||l[m]||d[m]||i;return r?n.createElement(f,a(a({ref:t},s),{},{components:r})):n.createElement(f,a({ref:t},s))}));function f(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=r.length,a=new Array(i);a[0]=m;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p[l]="string"==typeof e?e:o,a[1]=p;for(var u=2;u{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>a,default:()=>d,frontMatter:()=>i,metadata:()=>p,toc:()=>u});var n=r(7462),o=(r(7294),r(3905));const i={},a="Developer Environment Setup",p={unversionedId:"contribute/developer-env-setup",id:"contribute/developer-env-setup",title:"Developer Environment Setup",description:"To test Optimus including the integration with Airflow, as well as testing the runtime interactions, Optimus provides",source:"@site/docs/contribute/developer-env-setup.md",sourceDirName:"contribute",slug:"/contribute/developer-env-setup",permalink:"/optimus/docs/contribute/developer-env-setup",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/contribute/developer-env-setup.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Contributing",permalink:"/optimus/docs/contribute/contribution-process"},next:{title:"API",permalink:"/optimus/docs/reference/api"}},c={},u=[],s={toc:u},l="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(l,(0,n.Z)({},s,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"developer-environment-setup"},"Developer Environment Setup"),(0,o.kt)("p",null,"To test Optimus including the integration with Airflow, as well as testing the runtime interactions, Optimus provides\na way to simplify setting up the environment. Take a look and follow the guide in dev setup ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/optimus/tree/main/dev"},"section"),"."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/543eea9a.15be6dd8.js b/assets/js/543eea9a.b6af3155.js similarity index 98% rename from assets/js/543eea9a.15be6dd8.js rename to assets/js/543eea9a.b6af3155.js index 26d1c06b69..0fbbadcb89 100644 --- a/assets/js/543eea9a.15be6dd8.js +++ b/assets/js/543eea9a.b6af3155.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[1986],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>g});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),c=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=c(e.components);return a.createElement(s.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,s=e.parentName,p=i(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,g=u["".concat(s,".").concat(d)]||u[d]||m[d]||o;return n?a.createElement(g,l(l({ref:t},p),{},{components:n})):a.createElement(g,l({ref:t},p))}));function g(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,l=new Array(o);l[0]=d;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i[u]="string"==typeof e?e:r,l[1]=i;for(var c=2;c{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>m,frontMatter:()=>o,metadata:()=>i,toc:()=>c});var a=n(7462),r=(n(7294),n(3905));const o={},l="Manage BigQuery Resource",i={unversionedId:"client-guide/manage-bigquery-resource",id:"client-guide/manage-bigquery-resource",title:"Manage BigQuery Resource",description:"Below is the list of the resource types that Optimus supported:",source:"@site/docs/client-guide/manage-bigquery-resource.md",sourceDirName:"client-guide",slug:"/client-guide/manage-bigquery-resource",permalink:"/optimus/docs/client-guide/manage-bigquery-resource",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/manage-bigquery-resource.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Installing Plugin in Client",permalink:"/optimus/docs/client-guide/installing-plugin"},next:{title:"Create Job Specifications",permalink:"/optimus/docs/client-guide/create-job-specifications"}},s={},c=[{value:"Dataset",id:"dataset",level:2},{value:"Table",id:"table",level:2},{value:"View",id:"view",level:2},{value:"External Table",id:"external-table",level:2},{value:"Upload Resource Specifications",id:"upload-resource-specifications",level:2}],p={toc:c},u="wrapper";function m(e){let{components:t,...n}=e;return(0,r.kt)(u,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"manage-bigquery-resource"},"Manage BigQuery Resource"),(0,r.kt)("p",null,"Below is the list of the resource types that Optimus supported:"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"dataset"),(0,r.kt)("td",{parentName:"tr",align:null},"Resource name format: ","[project]",".","[dataset]"," ",(0,r.kt)("br",null)," Spec can includes: table_expiration, description")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"table"),(0,r.kt)("td",{parentName:"tr",align:null},"Resource name format: ","[project]",".","[dataset]",".","[table]"," ",(0,r.kt)("br",null)," Spec can includes: schema, partition, cluster, description")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"view"),(0,r.kt)("td",{parentName:"tr",align:null},"Resource name format: ","[project]",".","[dataset]",".","[view]"," ",(0,r.kt)("br",null)," Spec can includes: view_query, description")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"external_table"),(0,r.kt)("td",{parentName:"tr",align:null},"Resource name format: ","[project]",".","[dataset]",".","[table]"," ",(0,r.kt)("br",null)," Spec can include: schema, source, description")))),(0,r.kt)("p",null,"You can create any of the above jobs using the same following format:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource create\n")),(0,r.kt)("p",null,"Make sure to put the correct resource type as you are intended. Once you fill in the command prompt questions, Optimus will create a file (resource.yaml) in the configured datastore directory. Below is an example of each of the type\u2019s resource specifications."),(0,r.kt)("h2",{id:"dataset"},"Dataset"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground\ntype: dataset\nlabels:\n usage: documentation\n owner: optimus\nspec:\n description: "example description"\n table_expiration: 24 # in hours\n')),(0,r.kt)("h2",{id:"table"},"Table"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground.sample_table\ntype: table\nlabels:\n usage: documentation\n owner: optimus\nspec:\n description: "example description"\n schema:\n - name: column1\n type: INTEGER\n - name: column2\n type: TIMESTAMP\n description: "example field 2"\n mode: required # (repeated/required/nullable), default: nullable\n - name: column3\n type: STRUCT\n schema: # nested struct schema\n - name: column_a_1\n type: STRING\n cluster:\n using: [column1]\n partition: # leave empty as {} to partition by ingestion time\n field: column2 # column name\n type: day # day/hour, default: day\n')),(0,r.kt)("h2",{id:"view"},"View"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground.sample_view\ntype: view\nlabels:\n usage: documentation\n owner: optimus\nspec:\n description: "example description"\n view_query: |\n Select * from sample-project.playground.sample_table\n')),(0,r.kt)("h2",{id:"external-table"},"External Table"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground.sample_table\ntype: external_table\nlabels:\n usage: documentation\n owner: optimus\nspec:\n description: "example description"\n schema:\n - name: column1\n type: INTEGER\n - name: column2\n type: TIMESTAMP\n description: "example field 2"\n source:\n type: google_sheets\n uris:\n - https://docs.google.com/spreadsheets/d/spreadsheet_id\n config:\n range: Sheet1!A1:B4 # Range of data to be ingested in the format of [Sheet Name]![Cell Range]\n skip_leading_rows: 1 # Row of records to skip\n')),(0,r.kt)("h2",{id:"upload-resource-specifications"},"Upload Resource Specifications"),(0,r.kt)("p",null,"Once the resource specifications are ready, you can upload all resource specifications using the below command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource upload-all --verbose\n")),(0,r.kt)("p",null,"The above command will try to compare the incoming resources to the existing resources in the server. It will create\na new resource if it does not exist yet, and modify it if exists, but will not delete any resources. Optimus does not\nsupport BigQuery resource deletion nor the resource record in the Optimus server itself yet."))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[1986],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>g});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),c=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},p=function(e){var t=c(e.components);return a.createElement(s.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,s=e.parentName,p=i(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,g=u["".concat(s,".").concat(d)]||u[d]||m[d]||o;return n?a.createElement(g,l(l({ref:t},p),{},{components:n})):a.createElement(g,l({ref:t},p))}));function g(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,l=new Array(o);l[0]=d;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i[u]="string"==typeof e?e:r,l[1]=i;for(var c=2;c{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>m,frontMatter:()=>o,metadata:()=>i,toc:()=>c});var a=n(7462),r=(n(7294),n(3905));const o={},l="Manage BigQuery Resource",i={unversionedId:"client-guide/manage-bigquery-resource",id:"client-guide/manage-bigquery-resource",title:"Manage BigQuery Resource",description:"Below is the list of the resource types that Optimus supported:",source:"@site/docs/client-guide/manage-bigquery-resource.md",sourceDirName:"client-guide",slug:"/client-guide/manage-bigquery-resource",permalink:"/optimus/docs/client-guide/manage-bigquery-resource",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/manage-bigquery-resource.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Installing Plugin in Client",permalink:"/optimus/docs/client-guide/installing-plugin"},next:{title:"Create Job Specifications",permalink:"/optimus/docs/client-guide/create-job-specifications"}},s={},c=[{value:"Dataset",id:"dataset",level:2},{value:"Table",id:"table",level:2},{value:"View",id:"view",level:2},{value:"External Table",id:"external-table",level:2},{value:"Upload Resource Specifications",id:"upload-resource-specifications",level:2}],p={toc:c},u="wrapper";function m(e){let{components:t,...n}=e;return(0,r.kt)(u,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"manage-bigquery-resource"},"Manage BigQuery Resource"),(0,r.kt)("p",null,"Below is the list of the resource types that Optimus supported:"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"dataset"),(0,r.kt)("td",{parentName:"tr",align:null},"Resource name format: ","[project]",".","[dataset]"," ",(0,r.kt)("br",null)," Spec can includes: table_expiration, description")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"table"),(0,r.kt)("td",{parentName:"tr",align:null},"Resource name format: ","[project]",".","[dataset]",".","[table]"," ",(0,r.kt)("br",null)," Spec can includes: schema, partition, cluster, description")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"view"),(0,r.kt)("td",{parentName:"tr",align:null},"Resource name format: ","[project]",".","[dataset]",".","[view]"," ",(0,r.kt)("br",null)," Spec can includes: view_query, description")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"external_table"),(0,r.kt)("td",{parentName:"tr",align:null},"Resource name format: ","[project]",".","[dataset]",".","[table]"," ",(0,r.kt)("br",null)," Spec can include: schema, source, description")))),(0,r.kt)("p",null,"You can create any of the above jobs using the same following format:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource create\n")),(0,r.kt)("p",null,"Make sure to put the correct resource type as you are intended. Once you fill in the command prompt questions, Optimus will create a file (resource.yaml) in the configured datastore directory. Below is an example of each of the type\u2019s resource specifications."),(0,r.kt)("h2",{id:"dataset"},"Dataset"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground\ntype: dataset\nlabels:\n usage: documentation\n owner: optimus\nspec:\n description: "example description"\n table_expiration: 24 # in hours\n')),(0,r.kt)("h2",{id:"table"},"Table"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground.sample_table\ntype: table\nlabels:\n usage: documentation\n owner: optimus\nspec:\n description: "example description"\n schema:\n - name: column1\n type: INTEGER\n - name: column2\n type: TIMESTAMP\n description: "example field 2"\n mode: required # (repeated/required/nullable), default: nullable\n - name: column3\n type: STRUCT\n schema: # nested struct schema\n - name: column_a_1\n type: STRING\n cluster:\n using: [column1]\n partition: # leave empty as {} to partition by ingestion time\n field: column2 # column name\n type: day # day/hour, default: day\n')),(0,r.kt)("h2",{id:"view"},"View"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground.sample_view\ntype: view\nlabels:\n usage: documentation\n owner: optimus\nspec:\n description: "example description"\n view_query: |\n Select * from sample-project.playground.sample_table\n')),(0,r.kt)("h2",{id:"external-table"},"External Table"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.playground.sample_table\ntype: external_table\nlabels:\n usage: documentation\n owner: optimus\nspec:\n description: "example description"\n schema:\n - name: column1\n type: INTEGER\n - name: column2\n type: TIMESTAMP\n description: "example field 2"\n source:\n type: google_sheets\n uris:\n - https://docs.google.com/spreadsheets/d/spreadsheet_id\n config:\n range: Sheet1!A1:B4 # Range of data to be ingested in the format of [Sheet Name]![Cell Range]\n skip_leading_rows: 1 # Row of records to skip\n')),(0,r.kt)("h2",{id:"upload-resource-specifications"},"Upload Resource Specifications"),(0,r.kt)("p",null,"Once the resource specifications are ready, you can upload all resource specifications using the below command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource upload-all --verbose\n")),(0,r.kt)("p",null,"The above command will try to compare the incoming resources to the existing resources in the server. It will create\na new resource if it does not exist yet, and modify it if exists, but will not delete any resources. Optimus does not\nsupport BigQuery resource deletion nor the resource record in the Optimus server itself yet."))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/54f44165.30190704.js b/assets/js/54f44165.0f14f797.js similarity index 98% rename from assets/js/54f44165.30190704.js rename to assets/js/54f44165.0f14f797.js index 6dde358156..5cf7c305fb 100644 --- a/assets/js/54f44165.30190704.js +++ b/assets/js/54f44165.0f14f797.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[152],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>d});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),u=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=u(e.components);return a.createElement(s.Provider,{value:t},e.children)},c="mdxType",g={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},m=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),c=u(n),m=r,d=c["".concat(s,".").concat(m)]||c[m]||g[m]||l;return n?a.createElement(d,i(i({ref:t},p),{},{components:n})):a.createElement(d,i({ref:t},p))}));function d(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=n.length,i=new Array(l);i[0]=m;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o[c]="string"==typeof e?e:r,i[1]=o;for(var u=2;u{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>g,frontMatter:()=>l,metadata:()=>o,toc:()=>u});var a=n(7462),r=(n(7294),n(3905));const l={},i="Installation",o={unversionedId:"getting-started/installation",id:"getting-started/installation",title:"Installation",description:"Installing Optimus on any system is straight forward. There are several approaches to install Optimus:",source:"@site/docs/getting-started/installation.md",sourceDirName:"getting-started",slug:"/getting-started/installation",permalink:"/optimus/docs/getting-started/installation",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/getting-started/installation.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Introduction",permalink:"/optimus/docs/introduction"},next:{title:"Quickstart",permalink:"/optimus/docs/getting-started/quick-start"}},s={},u=[{value:"Using a Pre-built Binary",id:"using-a-pre-built-binary",level:2},{value:"Installing with Package Manager",id:"installing-with-package-manager",level:2},{value:"Installing using Docker",id:"installing-using-docker",level:2},{value:"Installing from Source",id:"installing-from-source",level:2},{value:"Prerequisites",id:"prerequisites",level:3},{value:"Build",id:"build",level:3}],p={toc:u},c="wrapper";function g(e){let{components:t,...n}=e;return(0,r.kt)(c,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"installation"},"Installation"),(0,r.kt)("p",null,"Installing Optimus on any system is straight forward. There are several approaches to install Optimus:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Using a pre-built binary"),(0,r.kt)("li",{parentName:"ul"},"Installing with package manager"),(0,r.kt)("li",{parentName:"ul"},"Installing with Docker"),(0,r.kt)("li",{parentName:"ul"},"Installing from source")),(0,r.kt)("h2",{id:"using-a-pre-built-binary"},"Using a Pre-built Binary"),(0,r.kt)("p",null,"The client and server binaries are downloadable at the ",(0,r.kt)("a",{parentName:"p",href:"https://github.com/goto/optimus/releases"},"releases")," section."),(0,r.kt)("p",null,"Once installed, you should be able to run:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus version\n")),(0,r.kt)("h2",{id:"installing-with-package-manager"},"Installing with Package Manager"),(0,r.kt)("p",null,"For macOS, you can install Optimus using homebrew:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ brew install goto/tap/optimus\n$ optimus version\n")),(0,r.kt)("h2",{id:"installing-using-docker"},"Installing using Docker"),(0,r.kt)("p",null,"To pull latest image:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ docker pull goto/optimus:latest\n")),(0,r.kt)("p",null,"To pull specific image:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ docker pull goto/optimus:0.6.0\n")),(0,r.kt)("h2",{id:"installing-from-source"},"Installing from Source"),(0,r.kt)("h3",{id:"prerequisites"},"Prerequisites"),(0,r.kt)("p",null,"Optimus requires the following dependencies:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Golang (version 1.18 or above)"),(0,r.kt)("li",{parentName:"ul"},"Git")),(0,r.kt)("h3",{id:"build"},"Build"),(0,r.kt)("p",null,"Run the following commands to compile optimus from source"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ git clone git@github.com:goto/optimus.git\n$ cd optimus\n$ make build\n")),(0,r.kt)("p",null,"Use the following command to test"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus version\n")))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[152],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>d});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),u=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=u(e.components);return a.createElement(s.Provider,{value:t},e.children)},c="mdxType",g={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},m=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),c=u(n),m=r,d=c["".concat(s,".").concat(m)]||c[m]||g[m]||l;return n?a.createElement(d,i(i({ref:t},p),{},{components:n})):a.createElement(d,i({ref:t},p))}));function d(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=n.length,i=new Array(l);i[0]=m;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o[c]="string"==typeof e?e:r,i[1]=o;for(var u=2;u{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>g,frontMatter:()=>l,metadata:()=>o,toc:()=>u});var a=n(7462),r=(n(7294),n(3905));const l={},i="Installation",o={unversionedId:"getting-started/installation",id:"getting-started/installation",title:"Installation",description:"Installing Optimus on any system is straight forward. There are several approaches to install Optimus:",source:"@site/docs/getting-started/installation.md",sourceDirName:"getting-started",slug:"/getting-started/installation",permalink:"/optimus/docs/getting-started/installation",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/getting-started/installation.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Introduction",permalink:"/optimus/docs/introduction"},next:{title:"Quickstart",permalink:"/optimus/docs/getting-started/quick-start"}},s={},u=[{value:"Using a Pre-built Binary",id:"using-a-pre-built-binary",level:2},{value:"Installing with Package Manager",id:"installing-with-package-manager",level:2},{value:"Installing using Docker",id:"installing-using-docker",level:2},{value:"Installing from Source",id:"installing-from-source",level:2},{value:"Prerequisites",id:"prerequisites",level:3},{value:"Build",id:"build",level:3}],p={toc:u},c="wrapper";function g(e){let{components:t,...n}=e;return(0,r.kt)(c,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"installation"},"Installation"),(0,r.kt)("p",null,"Installing Optimus on any system is straight forward. There are several approaches to install Optimus:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Using a pre-built binary"),(0,r.kt)("li",{parentName:"ul"},"Installing with package manager"),(0,r.kt)("li",{parentName:"ul"},"Installing with Docker"),(0,r.kt)("li",{parentName:"ul"},"Installing from source")),(0,r.kt)("h2",{id:"using-a-pre-built-binary"},"Using a Pre-built Binary"),(0,r.kt)("p",null,"The client and server binaries are downloadable at the ",(0,r.kt)("a",{parentName:"p",href:"https://github.com/goto/optimus/releases"},"releases")," section."),(0,r.kt)("p",null,"Once installed, you should be able to run:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus version\n")),(0,r.kt)("h2",{id:"installing-with-package-manager"},"Installing with Package Manager"),(0,r.kt)("p",null,"For macOS, you can install Optimus using homebrew:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ brew install goto/tap/optimus\n$ optimus version\n")),(0,r.kt)("h2",{id:"installing-using-docker"},"Installing using Docker"),(0,r.kt)("p",null,"To pull latest image:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ docker pull goto/optimus:latest\n")),(0,r.kt)("p",null,"To pull specific image:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ docker pull goto/optimus:0.6.0\n")),(0,r.kt)("h2",{id:"installing-from-source"},"Installing from Source"),(0,r.kt)("h3",{id:"prerequisites"},"Prerequisites"),(0,r.kt)("p",null,"Optimus requires the following dependencies:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Golang (version 1.18 or above)"),(0,r.kt)("li",{parentName:"ul"},"Git")),(0,r.kt)("h3",{id:"build"},"Build"),(0,r.kt)("p",null,"Run the following commands to compile optimus from source"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ git clone git@github.com:goto/optimus.git\n$ cd optimus\n$ make build\n")),(0,r.kt)("p",null,"Use the following command to test"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus version\n")))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/553314f8.d58c9075.js b/assets/js/553314f8.b1d4357b.js similarity index 99% rename from assets/js/553314f8.d58c9075.js rename to assets/js/553314f8.b1d4357b.js index cd82e4c974..3894ecc4d1 100644 --- a/assets/js/553314f8.d58c9075.js +++ b/assets/js/553314f8.b1d4357b.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2397],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>d});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function o(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},c="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},g=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=p(n),g=a,d=c["".concat(s,".").concat(g)]||c[g]||m[g]||i;return n?r.createElement(d,o(o({ref:t},u),{},{components:n})):r.createElement(d,o({ref:t},u))}));function d(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,o=new Array(i);o[0]=g;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[c]="string"==typeof e?e:a,o[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>o,default:()=>m,frontMatter:()=>i,metadata:()=>l,toc:()=>p});var r=n(7462),a=(n(7294),n(3905));const i={},o="Starting Optimus Server",l={unversionedId:"server-guide/starting-optimus-server",id:"server-guide/starting-optimus-server",title:"Starting Optimus Server",description:"Starting a server requires server configuration, which can be loaded from file (use --config flag), environment",source:"@site/docs/server-guide/starting-optimus-server.md",sourceDirName:"server-guide",slug:"/server-guide/starting-optimus-server",permalink:"/optimus/docs/server-guide/starting-optimus-server",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/server-guide/starting-optimus-server.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Installing Plugins",permalink:"/optimus/docs/server-guide/installing-plugins"},next:{title:"DB Migrations",permalink:"/optimus/docs/server-guide/db-migrations"}},s={},p=[],u={toc:p},c="wrapper";function m(e){let{components:t,...n}=e;return(0,a.kt)(c,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"starting-optimus-server"},"Starting Optimus Server"),(0,a.kt)("p",null,"Starting a server requires server configuration, which can be loaded from file (use --config flag), environment\nvariable ",(0,a.kt)("inlineCode",{parentName:"p"},"OPTIMUS_[CONFIGNAME]"),", or config.yaml file in Optimus binary directory."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"1. Using --config flag")),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus serve --config /path/to/config/file.yaml\n")),(0,a.kt)("p",null,"If you specify the configuration file using the --config flag, then any configs defined in the env variable and default\nconfig.yaml from the Optimus binary directory will not be loaded."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"2. Using environment variable")),(0,a.kt)("p",null,"All the configs can be passed as environment variables using ",(0,a.kt)("inlineCode",{parentName:"p"},"OPTIMUS_[CONFIG_NAME]")," convention. ","[CONFIG_NAME]"," is the\nkey name of config using ",(0,a.kt)("inlineCode",{parentName:"p"},"_")," as the path delimiter to concatenate between keys."),(0,a.kt)("p",null,"For example, to use the environment variable, assuming the following configuration layout:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-yaml"},"version: 1\nserve:\n port: 9100\n app_key: randomhash\n")),(0,a.kt)("p",null,"Here is the corresponding environment variable for the above"),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Configuration key"),(0,a.kt)("th",{parentName:"tr",align:null},"Environment variable"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"version"),(0,a.kt)("td",{parentName:"tr",align:null},"OPTIMUS_VERSION")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"serve.port"),(0,a.kt)("td",{parentName:"tr",align:null},"OPTIMUS_PORT")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"serve.app_key"),(0,a.kt)("td",{parentName:"tr",align:null},"OPTIMUS_SERVE_APP_KEY")))),(0,a.kt)("p",null,"Set the env variable using export"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ export OPTIMUS_PORT=9100\n")),(0,a.kt)("p",null,"Note: If you specify the env variable and you also have config.yaml in the Optimus binary directory, then any configs\nfrom the env variable will override the configs defined in config.yaml in Optimus binary directory."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"3. Using default config.yaml from Optimus binary directory")),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ which optimus\n/usr/local/bin/optimus\n")),(0,a.kt)("p",null,"So the config.yaml file can be loaded on /usr/local/bin/config.yaml"))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2397],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>d});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function o(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},c="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},g=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=p(n),g=a,d=c["".concat(s,".").concat(g)]||c[g]||m[g]||i;return n?r.createElement(d,o(o({ref:t},u),{},{components:n})):r.createElement(d,o({ref:t},u))}));function d(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,o=new Array(i);o[0]=g;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[c]="string"==typeof e?e:a,o[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>o,default:()=>m,frontMatter:()=>i,metadata:()=>l,toc:()=>p});var r=n(7462),a=(n(7294),n(3905));const i={},o="Starting Optimus Server",l={unversionedId:"server-guide/starting-optimus-server",id:"server-guide/starting-optimus-server",title:"Starting Optimus Server",description:"Starting a server requires server configuration, which can be loaded from file (use --config flag), environment",source:"@site/docs/server-guide/starting-optimus-server.md",sourceDirName:"server-guide",slug:"/server-guide/starting-optimus-server",permalink:"/optimus/docs/server-guide/starting-optimus-server",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/server-guide/starting-optimus-server.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Installing Plugins",permalink:"/optimus/docs/server-guide/installing-plugins"},next:{title:"DB Migrations",permalink:"/optimus/docs/server-guide/db-migrations"}},s={},p=[],u={toc:p},c="wrapper";function m(e){let{components:t,...n}=e;return(0,a.kt)(c,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"starting-optimus-server"},"Starting Optimus Server"),(0,a.kt)("p",null,"Starting a server requires server configuration, which can be loaded from file (use --config flag), environment\nvariable ",(0,a.kt)("inlineCode",{parentName:"p"},"OPTIMUS_[CONFIGNAME]"),", or config.yaml file in Optimus binary directory."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"1. Using --config flag")),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus serve --config /path/to/config/file.yaml\n")),(0,a.kt)("p",null,"If you specify the configuration file using the --config flag, then any configs defined in the env variable and default\nconfig.yaml from the Optimus binary directory will not be loaded."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"2. Using environment variable")),(0,a.kt)("p",null,"All the configs can be passed as environment variables using ",(0,a.kt)("inlineCode",{parentName:"p"},"OPTIMUS_[CONFIG_NAME]")," convention. ","[CONFIG_NAME]"," is the\nkey name of config using ",(0,a.kt)("inlineCode",{parentName:"p"},"_")," as the path delimiter to concatenate between keys."),(0,a.kt)("p",null,"For example, to use the environment variable, assuming the following configuration layout:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-yaml"},"version: 1\nserve:\n port: 9100\n app_key: randomhash\n")),(0,a.kt)("p",null,"Here is the corresponding environment variable for the above"),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Configuration key"),(0,a.kt)("th",{parentName:"tr",align:null},"Environment variable"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"version"),(0,a.kt)("td",{parentName:"tr",align:null},"OPTIMUS_VERSION")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"serve.port"),(0,a.kt)("td",{parentName:"tr",align:null},"OPTIMUS_PORT")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"serve.app_key"),(0,a.kt)("td",{parentName:"tr",align:null},"OPTIMUS_SERVE_APP_KEY")))),(0,a.kt)("p",null,"Set the env variable using export"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ export OPTIMUS_PORT=9100\n")),(0,a.kt)("p",null,"Note: If you specify the env variable and you also have config.yaml in the Optimus binary directory, then any configs\nfrom the env variable will override the configs defined in config.yaml in Optimus binary directory."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"3. Using default config.yaml from Optimus binary directory")),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ which optimus\n/usr/local/bin/optimus\n")),(0,a.kt)("p",null,"So the config.yaml file can be loaded on /usr/local/bin/config.yaml"))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/588bd741.eb9469ea.js b/assets/js/588bd741.ed5a26d8.js similarity index 99% rename from assets/js/588bd741.eb9469ea.js rename to assets/js/588bd741.ed5a26d8.js index 0776289614..a8aba312ef 100644 --- a/assets/js/588bd741.eb9469ea.js +++ b/assets/js/588bd741.ed5a26d8.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[4053],{3905:(e,t,a)=>{a.d(t,{Zo:()=>c,kt:()=>h});var n=a(7294);function s(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function o(e){for(var t=1;t=0||(s[a]=e[a]);return s}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(s[a]=e[a])}return s}var i=n.createContext({}),p=function(e){var t=n.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},c=function(e){var t=p(e.components);return n.createElement(i.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var a=e.components,s=e.mdxType,r=e.originalType,i=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=p(a),d=s,h=u["".concat(i,".").concat(d)]||u[d]||m[d]||r;return a?n.createElement(h,o(o({ref:t},c),{},{components:a})):n.createElement(h,o({ref:t},c))}));function h(e,t){var a=arguments,s=t&&t.mdxType;if("string"==typeof e||s){var r=a.length,o=new Array(r);o[0]=d;var l={};for(var i in t)hasOwnProperty.call(t,i)&&(l[i]=t[i]);l.originalType=e,l[u]="string"==typeof e?e:s,o[1]=l;for(var p=2;p{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>o,default:()=>m,frontMatter:()=>r,metadata:()=>l,toc:()=>p});var n=a(7462),s=(a(7294),a(3905));const r={},o="Quickstart",l={unversionedId:"getting-started/quick-start",id:"getting-started/quick-start",title:"Quickstart",description:"This quick start will guide you to try out Optimus fast without getting into many details. As part of this, you will",source:"@site/docs/getting-started/quick-start.md",sourceDirName:"getting-started",slug:"/getting-started/quick-start",permalink:"/optimus/docs/getting-started/quick-start",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/getting-started/quick-start.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Installation",permalink:"/optimus/docs/getting-started/installation"},next:{title:"Architecture",permalink:"/optimus/docs/concepts/architecture"}},i={},p=[{value:"Prerequisite",id:"prerequisite",level:2},{value:"Step 1: Start Server",id:"step-1-start-server",level:2},{value:"Start Server",id:"start-server",level:3},{value:"Step 2: Connect Client With Server",id:"step-2-connect-client-with-server",level:2},{value:"Step 3: Create BigQuery resource",id:"step-3-create-bigquery-resource",level:2},{value:"Step 4: Create & Deploy Job",id:"step-4-create--deploy-job",level:2}],c={toc:p},u="wrapper";function m(e){let{components:t,...a}=e;return(0,s.kt)(u,(0,n.Z)({},c,a,{components:t,mdxType:"MDXLayout"}),(0,s.kt)("h1",{id:"quickstart"},"Quickstart"),(0,s.kt)("p",null,"This quick start will guide you to try out Optimus fast without getting into many details. As part of this, you will\nbe provided with step-by-step instructions to start Optimus server, connect Optimus client with server, create\nBigQuery resource through Optimus, create BigQuery to BigQuery job, and deploy it."),(0,s.kt)("h2",{id:"prerequisite"},"Prerequisite"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},"Docker or a local installation of Optimus."),(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("a",{parentName:"li",href:"https://www.postgresql.org/download/"},"Postgres")," database."),(0,s.kt)("li",{parentName:"ul"},"BigQuery project"),(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("a",{parentName:"li",href:"https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html"},"Airflow"),(0,s.kt)("ul",{parentName:"li"},(0,s.kt)("li",{parentName:"ul"},"This is not mandatory to complete the quick start, but needed for scheduling jobs.")))),(0,s.kt)("h2",{id:"step-1-start-server"},"Step 1: Start Server"),(0,s.kt)("p",null,"Start server with GOTO\u2019s BigQuery to BigQuery ",(0,s.kt)("a",{parentName:"p",href:"https://github.com/goto/transformers"},"plugin"),"."),(0,s.kt)("p",null,"Create a config.yaml file:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},"version: 1\n\nlog:\n level: debug\n\nserve:\n port: 9100\n host: localhost\n ingress_host: localhost:9100\n app_key:\n db:\n dsn: postgres://@localhost:5432/dbname?sslmode=disable\n\nplugin:\n artifacts:\n - https://github.com/goto/transformers/releases/download/v0.3.15/transformers_0.3.15_macos_x86_64.tar.gz\n")),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: make sure you put artifacts link that suitable to your system.")),(0,s.kt)("h3",{id:"start-server"},"Start Server"),(0,s.kt)("p",null,"With the config.yaml available in the same working directory, you can start server by running:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus serve --install-plugins\n")),(0,s.kt)("p",null,"This will automatically install the plugins as specified in your server configuration."),(0,s.kt)("h2",{id:"step-2-connect-client-with-server"},"Step 2: Connect Client With Server"),(0,s.kt)("p",null,"Go to the directory where you want to have your Optimus specifications. Create client configuration by using\noptimus ",(0,s.kt)("inlineCode",{parentName:"p"},"init")," command. An interactive questionnaire will be presented, such as below:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus init\n\n? What is the Optimus service host? localhost:9100\n? What is the Optimus project name? sample_project\n? What is the namespace name? sample_namespace\n? What is the type of data store for this namespace? bigquery\n? Do you want to add another namespace? No\nClient config is initialized successfully\n")),(0,s.kt)("p",null,"After running the init command, Optimus client config will be configured. Along with it, the directories for the chosen\nnamespaces, including the subdirectories for jobs and resources will be created with the following structure:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"sample_project\n\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n\u2502 \u2514\u2500\u2500 resources\n\u2514\u2500\u2500 optimus.yaml\n")),(0,s.kt)("p",null,"Below is the client configuration that has been generated:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nlog:\n level: INFO\n format: ""\nhost: localhost:9100\nproject:\n name: sample_project\n config: {}\nnamespaces:\n- name: sample_namespace\n config: {}\n job:\n path: sample_namespace/jobs\n datastore:\n - type: bigquery\n path: sample_namespace/resources\n backup: {}\n')),(0,s.kt)("p",null,"Let\u2019s add ",(0,s.kt)("inlineCode",{parentName:"p"},"storage_path")," project configuration that is needed to store the result of job compilation and\n",(0,s.kt)("inlineCode",{parentName:"p"},"scheduler_host")," which is needed for compilation."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},"project:\n name: sample_project\n config:\n storage_path: file:///Users/sample_user/optimus/sample_project/compiled\n scheduler_host: http://sample-host\n")),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: storage path is the location where airflow is reading its dags from.")),(0,s.kt)("p",null,"Now, let's register ",(0,s.kt)("inlineCode",{parentName:"p"},"sample_project")," and ",(0,s.kt)("inlineCode",{parentName:"p"},"sample_namespace")," to your Optimus server."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project register --with-namespaces\n")),(0,s.kt)("p",null,"You can verify if the project has been registered successfully by running this command:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project describe\n")),(0,s.kt)("h2",{id:"step-3-create-bigquery-resource"},"Step 3: Create BigQuery resource"),(0,s.kt)("p",null,"Before creating BigQuery resources, make sure your Optimus server has access to your BQ project by adding a\n",(0,s.kt)("inlineCode",{parentName:"p"},"BQ_SERVICE_ACCOUNT")," secret."),(0,s.kt)("p",null,"Assume you have your service account json file in the same directory (project directory), create the secret using the\nfollowing command. Make sure the service account that you are using is authorized to create tables."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set BQ_SERVICE_ACCOUNT --file service_account.json\n")),(0,s.kt)("p",null,"Check whether the secret has been registered successfully by running this command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret list\n")),(0,s.kt)("p",null,"Now, let\u2019s create a resource using the following interactive command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource create\n\n? Please choose the namespace: sample_namespace\n? What is the resource name? sample-project.sample_namespace.table1\n? What is the resource type? table\n? Provide new directory name to create for this spec? [sample_namespace/resources] sample-project.sample_namespace.table1\n\nResource spec [sample-project.sample_namespace.table1] is created successfully\n")),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: resource name should be unique within the project. Take a look at the complete guide on how to create resource\n",(0,s.kt)("a",{parentName:"em",href:"/optimus/docs/client-guide/manage-bigquery-resource"},"here")," if needed.")),(0,s.kt)("p",null,"After running the command, the resource specification file will be automatically created in the following directory:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"sample_project\n\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n\u2502 \u2514\u2500\u2500 resources\n| \u2514\u2500\u2500 sample-project.sample_namespace.table1\n| \u2514\u2500\u2500 resource.yaml\n\u2514\u2500\u2500 optimus.yaml\n")),(0,s.kt)("p",null,"Let\u2019s open the resource.yaml file and add additional spec details as follows:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.sample_namespace.table1\ntype: table\nlabels: {}\nspec:\n description: "sample optimus quick start table"\n schema:\n - name: sample_day\n type: STRING\n mode: NULLABLE\n - name: sample_timestamp\n type: TIMESTAMP\n mode: NULLABLE\n')),(0,s.kt)("p",null,"Now that resource specification is complete, let\u2019s deploy this to the Optimus server and it will create the resource\nin BigQuery."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource upload-all --verbose\n\n> Validating namespaces\nnamespace validation finished!\n\n> Uploading all resources for namespaces [sample_namespace]\n> Deploying bigquery resources for namespace [sample_namespace]\n> Receiving responses:\n[success] sample-project.sample_namespace.table1\nresources with namespace [sample_namespace] are deployed successfully\nfinished uploading resource specifications to server!\n")),(0,s.kt)("h2",{id:"step-4-create--deploy-job"},"Step 4: Create & Deploy Job"),(0,s.kt)("p",null,"Sync plugins to your local for optimus to provide an interactive UI to add jobs, this is a prerequisite before\ncreating any jobs."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus plugin sync\n")),(0,s.kt)("p",null,"Let\u2019s verify if the plugin has been synced properly by running below command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus version\n")),(0,s.kt)("p",null,"You should find ",(0,s.kt)("inlineCode",{parentName:"p"},"bq2bq")," plugin in the list of discovered plugins."),(0,s.kt)("p",null,"To create a job, we need to provide a job specification. Let\u2019s create one using the interactive optimus job command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job create \n? Please choose the namespace: sample_namespace\n? Provide new directory name to create for this spec? [.] sample-project.sample_namespace.table1\n? What is the job name? sample-project.sample_namespace.table1\n? Who is the owner of this job? sample_owner\n? Select task to run? bq2bq\n? Specify the schedule start date 2023-01-26\n? Specify the schedule interval (in crontab notation) 0 2 * * *\n? Window truncate to: d\n? Window offset: 0\n? Window size: 24h\n? Project ID sample-project\n? Dataset Name sample_namespace\n? Table ID table1\n? Load method to use on destination REPLACE\nJob successfully created at sample-project.sample_namespace.table1\n")),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: take a look at the details of job creation ",(0,s.kt)("a",{parentName:"em",href:"/optimus/docs/client-guide/create-job-specifications"},"here"),".")),(0,s.kt)("p",null,"After running the job create command, the job specification file and assets directory are created in the following directory."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n| \u2514\u2500\u2500 sample-project.sample_namespace.table1\n| \u2514\u2500\u2500 assets\n| \u2514\u2500\u2500 query.sql\n| \u2514\u2500\u2500 job.yaml\n\u2502 \u2514\u2500\u2500 resources\n| \u2514\u2500\u2500 sample-project.sample_namespace.table1\n| \u2514\u2500\u2500 resource.yaml\n\u2514\u2500\u2500 optimus.yaml\n")),(0,s.kt)("p",null,"For BQ2BQ job, the core transformation logic lies in ",(0,s.kt)("inlineCode",{parentName:"p"},"assets/query.sql"),". Let\u2019s modify the query to the following script:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-sql"},'SELECT\nFORMAT_DATE(\'%A\', CAST("{{ .DSTART }}" AS TIMESTAMP)) AS `sample_day`,\nCAST("{{ .DSTART }}" AS TIMESTAMP) AS `sample_timestamp`;\n')),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: take a look at Optimus\u2019 supported macros ",(0,s.kt)("a",{parentName:"em",href:"/optimus/docs/concepts/macros"},"here"),".")),(0,s.kt)("p",null,"Let\u2019s also verify the generated job.yaml file."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.sample_namespace.table1\nowner: sample_owner\nschedule:\n start_date: "2023-01-26"\n interval: 0 2 * * *\nbehavior:\n depends_on_past: false\ntask:\n name: bq2bq\n config:\n DATASET: sample_namespace\n LOAD_METHOD: REPLACE\n PROJECT: sample-project\n SQL_TYPE: STANDARD\n TABLE: table1\n window:\n size: 24h\n offset: "0"\n truncate_to: d\nlabels:\n orchestrator: optimus\nhooks: []\ndependencies: []\n')),(0,s.kt)("p",null,"For this quick start, we are not adding any hooks, dependencies, or alert configurations. Take a look at the details\nof job specification and the possible options ",(0,s.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/create-job-specifications#understanding-the-job-specifications"},"here"),"."),(0,s.kt)("p",null,"Before proceeding, let\u2019s add the BQ_SERVICE_ACCOUNT secret in the task configuration."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},'task:\n name: bq2bq\n config:\n BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"\n DATASET: sample_namespace\n...\n')),(0,s.kt)("p",null,"Later, you can avoid having the secret specified in every single job specification by adding it in the parent yaml\nspecification instead. For more details, you can take a look ",(0,s.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/organizing-specifications"},"here"),"."),(0,s.kt)("p",null,"Now the job specification has been prepared, lets try to add it to the server by running this command:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job replace-all --verbose\n\n> Validating namespaces\nvalidation finished!\n\n> Replacing all jobs for namespaces [sample_namespace]\n> Receiving responses:\n[sample_namespace] received 1 job specs\n[sample_namespace] found 1 new, 0 modified, and 0 deleted job specs\n[sample_namespace] processing job job1\n[sample_namespace] successfully added 1 jobs\nreplace all job specifications finished!\n")),(0,s.kt)("p",null,"Above command will try to add/modify all job specifications found in your project. We are not providing registering\na single job through Optimus CLI, but it is possible to do so using API."),(0,s.kt)("p",null,"Now that the jobs has been registered to Optimus, let\u2019s compile and upload it to the scheduler by using the following command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus scheduler upload-all\n")),(0,s.kt)("p",null,"The command will try to compile your job specification to the DAG file. The result will be stored in the ",(0,s.kt)("inlineCode",{parentName:"p"},"storage_path"),"\nlocation as you have specified when configuring the optimus.yaml file."),(0,s.kt)("p",null,"Later, once you have Airflow ready and want to try out, this directory can be used as a source to be scheduled by Airflow."))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[4053],{3905:(e,t,a)=>{a.d(t,{Zo:()=>c,kt:()=>h});var n=a(7294);function s(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function o(e){for(var t=1;t=0||(s[a]=e[a]);return s}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(s[a]=e[a])}return s}var i=n.createContext({}),p=function(e){var t=n.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},c=function(e){var t=p(e.components);return n.createElement(i.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var a=e.components,s=e.mdxType,r=e.originalType,i=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=p(a),d=s,h=u["".concat(i,".").concat(d)]||u[d]||m[d]||r;return a?n.createElement(h,o(o({ref:t},c),{},{components:a})):n.createElement(h,o({ref:t},c))}));function h(e,t){var a=arguments,s=t&&t.mdxType;if("string"==typeof e||s){var r=a.length,o=new Array(r);o[0]=d;var l={};for(var i in t)hasOwnProperty.call(t,i)&&(l[i]=t[i]);l.originalType=e,l[u]="string"==typeof e?e:s,o[1]=l;for(var p=2;p{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>o,default:()=>m,frontMatter:()=>r,metadata:()=>l,toc:()=>p});var n=a(7462),s=(a(7294),a(3905));const r={},o="Quickstart",l={unversionedId:"getting-started/quick-start",id:"getting-started/quick-start",title:"Quickstart",description:"This quick start will guide you to try out Optimus fast without getting into many details. As part of this, you will",source:"@site/docs/getting-started/quick-start.md",sourceDirName:"getting-started",slug:"/getting-started/quick-start",permalink:"/optimus/docs/getting-started/quick-start",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/getting-started/quick-start.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Installation",permalink:"/optimus/docs/getting-started/installation"},next:{title:"Architecture",permalink:"/optimus/docs/concepts/architecture"}},i={},p=[{value:"Prerequisite",id:"prerequisite",level:2},{value:"Step 1: Start Server",id:"step-1-start-server",level:2},{value:"Start Server",id:"start-server",level:3},{value:"Step 2: Connect Client With Server",id:"step-2-connect-client-with-server",level:2},{value:"Step 3: Create BigQuery resource",id:"step-3-create-bigquery-resource",level:2},{value:"Step 4: Create & Deploy Job",id:"step-4-create--deploy-job",level:2}],c={toc:p},u="wrapper";function m(e){let{components:t,...a}=e;return(0,s.kt)(u,(0,n.Z)({},c,a,{components:t,mdxType:"MDXLayout"}),(0,s.kt)("h1",{id:"quickstart"},"Quickstart"),(0,s.kt)("p",null,"This quick start will guide you to try out Optimus fast without getting into many details. As part of this, you will\nbe provided with step-by-step instructions to start Optimus server, connect Optimus client with server, create\nBigQuery resource through Optimus, create BigQuery to BigQuery job, and deploy it."),(0,s.kt)("h2",{id:"prerequisite"},"Prerequisite"),(0,s.kt)("ul",null,(0,s.kt)("li",{parentName:"ul"},"Docker or a local installation of Optimus."),(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("a",{parentName:"li",href:"https://www.postgresql.org/download/"},"Postgres")," database."),(0,s.kt)("li",{parentName:"ul"},"BigQuery project"),(0,s.kt)("li",{parentName:"ul"},(0,s.kt)("a",{parentName:"li",href:"https://airflow.apache.org/docs/apache-airflow/stable/installation/index.html"},"Airflow"),(0,s.kt)("ul",{parentName:"li"},(0,s.kt)("li",{parentName:"ul"},"This is not mandatory to complete the quick start, but needed for scheduling jobs.")))),(0,s.kt)("h2",{id:"step-1-start-server"},"Step 1: Start Server"),(0,s.kt)("p",null,"Start server with GOTO\u2019s BigQuery to BigQuery ",(0,s.kt)("a",{parentName:"p",href:"https://github.com/goto/transformers"},"plugin"),"."),(0,s.kt)("p",null,"Create a config.yaml file:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},"version: 1\n\nlog:\n level: debug\n\nserve:\n port: 9100\n host: localhost\n ingress_host: localhost:9100\n app_key:\n db:\n dsn: postgres://@localhost:5432/dbname?sslmode=disable\n\nplugin:\n artifacts:\n - https://github.com/goto/transformers/releases/download/v0.3.15/transformers_0.3.15_macos_x86_64.tar.gz\n")),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: make sure you put artifacts link that suitable to your system.")),(0,s.kt)("h3",{id:"start-server"},"Start Server"),(0,s.kt)("p",null,"With the config.yaml available in the same working directory, you can start server by running:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus serve --install-plugins\n")),(0,s.kt)("p",null,"This will automatically install the plugins as specified in your server configuration."),(0,s.kt)("h2",{id:"step-2-connect-client-with-server"},"Step 2: Connect Client With Server"),(0,s.kt)("p",null,"Go to the directory where you want to have your Optimus specifications. Create client configuration by using\noptimus ",(0,s.kt)("inlineCode",{parentName:"p"},"init")," command. An interactive questionnaire will be presented, such as below:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus init\n\n? What is the Optimus service host? localhost:9100\n? What is the Optimus project name? sample_project\n? What is the namespace name? sample_namespace\n? What is the type of data store for this namespace? bigquery\n? Do you want to add another namespace? No\nClient config is initialized successfully\n")),(0,s.kt)("p",null,"After running the init command, Optimus client config will be configured. Along with it, the directories for the chosen\nnamespaces, including the subdirectories for jobs and resources will be created with the following structure:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"sample_project\n\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n\u2502 \u2514\u2500\u2500 resources\n\u2514\u2500\u2500 optimus.yaml\n")),(0,s.kt)("p",null,"Below is the client configuration that has been generated:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nlog:\n level: INFO\n format: ""\nhost: localhost:9100\nproject:\n name: sample_project\n config: {}\nnamespaces:\n- name: sample_namespace\n config: {}\n job:\n path: sample_namespace/jobs\n datastore:\n - type: bigquery\n path: sample_namespace/resources\n backup: {}\n')),(0,s.kt)("p",null,"Let\u2019s add ",(0,s.kt)("inlineCode",{parentName:"p"},"storage_path")," project configuration that is needed to store the result of job compilation and\n",(0,s.kt)("inlineCode",{parentName:"p"},"scheduler_host")," which is needed for compilation."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},"project:\n name: sample_project\n config:\n storage_path: file:///Users/sample_user/optimus/sample_project/compiled\n scheduler_host: http://sample-host\n")),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: storage path is the location where airflow is reading its dags from.")),(0,s.kt)("p",null,"Now, let's register ",(0,s.kt)("inlineCode",{parentName:"p"},"sample_project")," and ",(0,s.kt)("inlineCode",{parentName:"p"},"sample_namespace")," to your Optimus server."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project register --with-namespaces\n")),(0,s.kt)("p",null,"You can verify if the project has been registered successfully by running this command:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project describe\n")),(0,s.kt)("h2",{id:"step-3-create-bigquery-resource"},"Step 3: Create BigQuery resource"),(0,s.kt)("p",null,"Before creating BigQuery resources, make sure your Optimus server has access to your BQ project by adding a\n",(0,s.kt)("inlineCode",{parentName:"p"},"BQ_SERVICE_ACCOUNT")," secret."),(0,s.kt)("p",null,"Assume you have your service account json file in the same directory (project directory), create the secret using the\nfollowing command. Make sure the service account that you are using is authorized to create tables."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set BQ_SERVICE_ACCOUNT --file service_account.json\n")),(0,s.kt)("p",null,"Check whether the secret has been registered successfully by running this command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret list\n")),(0,s.kt)("p",null,"Now, let\u2019s create a resource using the following interactive command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource create\n\n? Please choose the namespace: sample_namespace\n? What is the resource name? sample-project.sample_namespace.table1\n? What is the resource type? table\n? Provide new directory name to create for this spec? [sample_namespace/resources] sample-project.sample_namespace.table1\n\nResource spec [sample-project.sample_namespace.table1] is created successfully\n")),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: resource name should be unique within the project. Take a look at the complete guide on how to create resource\n",(0,s.kt)("a",{parentName:"em",href:"/optimus/docs/client-guide/manage-bigquery-resource"},"here")," if needed.")),(0,s.kt)("p",null,"After running the command, the resource specification file will be automatically created in the following directory:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"sample_project\n\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n\u2502 \u2514\u2500\u2500 resources\n| \u2514\u2500\u2500 sample-project.sample_namespace.table1\n| \u2514\u2500\u2500 resource.yaml\n\u2514\u2500\u2500 optimus.yaml\n")),(0,s.kt)("p",null,"Let\u2019s open the resource.yaml file and add additional spec details as follows:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.sample_namespace.table1\ntype: table\nlabels: {}\nspec:\n description: "sample optimus quick start table"\n schema:\n - name: sample_day\n type: STRING\n mode: NULLABLE\n - name: sample_timestamp\n type: TIMESTAMP\n mode: NULLABLE\n')),(0,s.kt)("p",null,"Now that resource specification is complete, let\u2019s deploy this to the Optimus server and it will create the resource\nin BigQuery."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource upload-all --verbose\n\n> Validating namespaces\nnamespace validation finished!\n\n> Uploading all resources for namespaces [sample_namespace]\n> Deploying bigquery resources for namespace [sample_namespace]\n> Receiving responses:\n[success] sample-project.sample_namespace.table1\nresources with namespace [sample_namespace] are deployed successfully\nfinished uploading resource specifications to server!\n")),(0,s.kt)("h2",{id:"step-4-create--deploy-job"},"Step 4: Create & Deploy Job"),(0,s.kt)("p",null,"Sync plugins to your local for optimus to provide an interactive UI to add jobs, this is a prerequisite before\ncreating any jobs."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus plugin sync\n")),(0,s.kt)("p",null,"Let\u2019s verify if the plugin has been synced properly by running below command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus version\n")),(0,s.kt)("p",null,"You should find ",(0,s.kt)("inlineCode",{parentName:"p"},"bq2bq")," plugin in the list of discovered plugins."),(0,s.kt)("p",null,"To create a job, we need to provide a job specification. Let\u2019s create one using the interactive optimus job command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job create \n? Please choose the namespace: sample_namespace\n? Provide new directory name to create for this spec? [.] sample-project.sample_namespace.table1\n? What is the job name? sample-project.sample_namespace.table1\n? Who is the owner of this job? sample_owner\n? Select task to run? bq2bq\n? Specify the schedule start date 2023-01-26\n? Specify the schedule interval (in crontab notation) 0 2 * * *\n? Window truncate to: d\n? Window offset: 0\n? Window size: 24h\n? Project ID sample-project\n? Dataset Name sample_namespace\n? Table ID table1\n? Load method to use on destination REPLACE\nJob successfully created at sample-project.sample_namespace.table1\n")),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: take a look at the details of job creation ",(0,s.kt)("a",{parentName:"em",href:"/optimus/docs/client-guide/create-job-specifications"},"here"),".")),(0,s.kt)("p",null,"After running the job create command, the job specification file and assets directory are created in the following directory."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre"},"\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n| \u2514\u2500\u2500 sample-project.sample_namespace.table1\n| \u2514\u2500\u2500 assets\n| \u2514\u2500\u2500 query.sql\n| \u2514\u2500\u2500 job.yaml\n\u2502 \u2514\u2500\u2500 resources\n| \u2514\u2500\u2500 sample-project.sample_namespace.table1\n| \u2514\u2500\u2500 resource.yaml\n\u2514\u2500\u2500 optimus.yaml\n")),(0,s.kt)("p",null,"For BQ2BQ job, the core transformation logic lies in ",(0,s.kt)("inlineCode",{parentName:"p"},"assets/query.sql"),". Let\u2019s modify the query to the following script:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-sql"},'SELECT\nFORMAT_DATE(\'%A\', CAST("{{ .DSTART }}" AS TIMESTAMP)) AS `sample_day`,\nCAST("{{ .DSTART }}" AS TIMESTAMP) AS `sample_timestamp`;\n')),(0,s.kt)("p",null,(0,s.kt)("em",{parentName:"p"},"Note: take a look at Optimus\u2019 supported macros ",(0,s.kt)("a",{parentName:"em",href:"/optimus/docs/concepts/macros"},"here"),".")),(0,s.kt)("p",null,"Let\u2019s also verify the generated job.yaml file."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample-project.sample_namespace.table1\nowner: sample_owner\nschedule:\n start_date: "2023-01-26"\n interval: 0 2 * * *\nbehavior:\n depends_on_past: false\ntask:\n name: bq2bq\n config:\n DATASET: sample_namespace\n LOAD_METHOD: REPLACE\n PROJECT: sample-project\n SQL_TYPE: STANDARD\n TABLE: table1\n window:\n size: 24h\n offset: "0"\n truncate_to: d\nlabels:\n orchestrator: optimus\nhooks: []\ndependencies: []\n')),(0,s.kt)("p",null,"For this quick start, we are not adding any hooks, dependencies, or alert configurations. Take a look at the details\nof job specification and the possible options ",(0,s.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/create-job-specifications#understanding-the-job-specifications"},"here"),"."),(0,s.kt)("p",null,"Before proceeding, let\u2019s add the BQ_SERVICE_ACCOUNT secret in the task configuration."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-yaml"},'task:\n name: bq2bq\n config:\n BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"\n DATASET: sample_namespace\n...\n')),(0,s.kt)("p",null,"Later, you can avoid having the secret specified in every single job specification by adding it in the parent yaml\nspecification instead. For more details, you can take a look ",(0,s.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/organizing-specifications"},"here"),"."),(0,s.kt)("p",null,"Now the job specification has been prepared, lets try to add it to the server by running this command:"),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job replace-all --verbose\n\n> Validating namespaces\nvalidation finished!\n\n> Replacing all jobs for namespaces [sample_namespace]\n> Receiving responses:\n[sample_namespace] received 1 job specs\n[sample_namespace] found 1 new, 0 modified, and 0 deleted job specs\n[sample_namespace] processing job job1\n[sample_namespace] successfully added 1 jobs\nreplace all job specifications finished!\n")),(0,s.kt)("p",null,"Above command will try to add/modify all job specifications found in your project. We are not providing registering\na single job through Optimus CLI, but it is possible to do so using API."),(0,s.kt)("p",null,"Now that the jobs has been registered to Optimus, let\u2019s compile and upload it to the scheduler by using the following command."),(0,s.kt)("pre",null,(0,s.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus scheduler upload-all\n")),(0,s.kt)("p",null,"The command will try to compile your job specification to the DAG file. The result will be stored in the ",(0,s.kt)("inlineCode",{parentName:"p"},"storage_path"),"\nlocation as you have specified when configuring the optimus.yaml file."),(0,s.kt)("p",null,"Later, once you have Airflow ready and want to try out, this directory can be used as a source to be scheduled by Airflow."))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/5ec01bbe.75441a54.js b/assets/js/5ec01bbe.f7aa917b.js similarity index 99% rename from assets/js/5ec01bbe.75441a54.js rename to assets/js/5ec01bbe.f7aa917b.js index c24e26de50..7f04056dc7 100644 --- a/assets/js/5ec01bbe.75441a54.js +++ b/assets/js/5ec01bbe.f7aa917b.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[4648],{3905:(e,t,n)=>{n.d(t,{Zo:()=>m,kt:()=>h});var a=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var r=a.createContext({}),u=function(e){var t=a.useContext(r),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},m=function(e){var t=u(e.components);return a.createElement(r.Provider,{value:t},e.children)},p="mdxType",c={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,r=e.parentName,m=s(e,["components","mdxType","originalType","parentName"]),p=u(n),d=i,h=p["".concat(r,".").concat(d)]||p[d]||c[d]||o;return n?a.createElement(h,l(l({ref:t},m),{},{components:n})):a.createElement(h,l({ref:t},m))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,l=new Array(o);l[0]=d;var s={};for(var r in t)hasOwnProperty.call(t,r)&&(s[r]=t[r]);s.originalType=e,s[p]="string"==typeof e?e:i,l[1]=s;for(var u=2;u{n.r(t),n.d(t,{assets:()=>r,contentTitle:()=>l,default:()=>c,frontMatter:()=>o,metadata:()=>s,toc:()=>u});var a=n(7462),i=(n(7294),n(3905));const o={},l="Work with Extension",s={unversionedId:"client-guide/work-with-extension",id:"client-guide/work-with-extension",title:"Work with Extension",description:"Extension helps you to include third-party or arbitrary implementation as part of Optimus. Currently, the extension is",source:"@site/docs/client-guide/work-with-extension.md",sourceDirName:"client-guide",slug:"/client-guide/work-with-extension",permalink:"/optimus/docs/client-guide/work-with-extension",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/work-with-extension.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Replay a Job (Backfill)",permalink:"/optimus/docs/client-guide/replay-a-job"},next:{title:"Defining Scheduler Version",permalink:"/optimus/docs/client-guide/defining-scheduler-version"}},r={},u=[{value:"Warning",id:"warning",level:2},{value:"Limitation",id:"limitation",level:2},{value:"Creating",id:"creating",level:2},{value:"Commands",id:"commands",level:2},{value:"Installation",id:"installation",level:3},{value:"Executing",id:"executing",level:3},{value:"Activate",id:"activate",level:3},{value:"Describe",id:"describe",level:3},{value:"Rename",id:"rename",level:3},{value:"Uninstall",id:"uninstall",level:3},{value:"Upgrade",id:"upgrade",level:3}],m={toc:u},p="wrapper";function c(e){let{components:t,...n}=e;return(0,i.kt)(p,(0,a.Z)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"work-with-extension"},"Work with Extension"),(0,i.kt)("p",null,"Extension helps you to include third-party or arbitrary implementation as part of Optimus. Currently, the extension is\ndesigned for when the user is running it as CLI."),(0,i.kt)("h2",{id:"warning"},"Warning"),(0,i.kt)("p",null,"Extension is basically an executable file outside Optimus. ",(0,i.kt)("em",{parentName:"p"},"We do not guarantee whether an extension is safe or not"),".\nWe suggest checking the extension itself, whether it is safe to run in your local or not, before installing and running it."),(0,i.kt)("h2",{id:"limitation"},"Limitation"),(0,i.kt)("p",null,"Extension is designed to be similar to ",(0,i.kt)("a",{parentName:"p",href:"https://cli.github.com/manual/gh_extension"},"GitHub extension"),". However, since\nit's still in the early stage, some limitations are there."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"extension is only an executable file"),(0,i.kt)("li",{parentName:"ul"},"installation only looks at the GitHub asset according to the running system OS and Architecture"),(0,i.kt)("li",{parentName:"ul"},"convention for extension:",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"extension repository should follow optimus-extension-","[name of extension]"," (example: ",(0,i.kt)("a",{parentName:"li",href:"https://github.com/gojek/optimus-extension-valor"},"optimus-extension-valor"),")"),(0,i.kt)("li",{parentName:"ul"},"asset being considered is binary with suffix ...","[OS]","-","[ARC]"," (example: when installing valor, if the user's OS is "),(0,i.kt)("li",{parentName:"ul"},"Linux and the architecture is AMD64, then installation will consider valor_linux-amd64 as binary to be executed)")))),(0,i.kt)("h2",{id:"creating"},"Creating"),(0,i.kt)("p",null,"Extension is designed to be open. Anyone could create their own extension. And as long as it is available,\nanyone could install it. In order to create it, the following are the basic steps to do:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Decide the name of the extension, example: ",(0,i.kt)("em",{parentName:"li"},"valor")),(0,i.kt)("li",{parentName:"ul"},"Create a GitHub repository that follows the convention, example: ",(0,i.kt)("em",{parentName:"li"},"optimus-extension-valor")),(0,i.kt)("li",{parentName:"ul"},"Put some implementation and asset with the name based on the convention, example: ",(0,i.kt)("em",{parentName:"li"},"valor_linux-amd64, valor_darwin-amd64"),", and more."),(0,i.kt)("li",{parentName:"ul"},"Ensure it is available for anyone to download")),(0,i.kt)("h2",{id:"commands"},"Commands"),(0,i.kt)("p",null,"Optimus supports some commands to help operate on extension."),(0,i.kt)("h3",{id:"installation"},"Installation"),(0,i.kt)("p",null,"You can run the installation using Optimus sub-command install under the extension. In order to install an extension, run the following command:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension install REMOTE [flags]\n")),(0,i.kt)("p",null,"You can use the ",(0,i.kt)("em",{parentName:"p"},"--alias")," flag to change the command name, since by default, Optimus will try to figure it out by itself.\nAlthough, during this process, sometimes an extension name conflicts with the reserved commands. This flag helps to\nresolve that. But, do note that this flag cannot be used to rename an ",(0,i.kt)("em",{parentName:"p"},"installed")," extension. To do such a thing, check rename."),(0,i.kt)("p",null,"REMOTE is the Github remote path where to look for the extension. REMOTE can be in the form of"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"OWNER/PROJECT"),(0,i.kt)("li",{parentName:"ul"},"github.com/OWNER/PROJECT"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("a",{parentName:"li",href:"https://www.github.com/OWNER/PROJECT"},"https://www.github.com/OWNER/PROJECT"))),(0,i.kt)("p",null,"One example of such an extension is Valor. So, going back to the example above, installing it is like this:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension install gojek/optimus-extension-valor@v0.0.4\n")),(0,i.kt)("p",null,"or"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension install github.com/gojek/optimus-extension-valor@v0.0.4\n")),(0,i.kt)("p",null,"or"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension install https://github.com/gojek/optimus-extension-valor@v0.0.4\n")),(0,i.kt)("p",null,"Installation process is then in progress. If the installation is a success, the user can show it by running:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus --help\n")),(0,i.kt)("p",null,"A new command named after the extension will be available. For example, if the extension name is ",(0,i.kt)("em",{parentName:"p"},"optimus-extension-valor"),",\nthen by default the command named valor will be available. If the user wishes to change it, they can use ",(0,i.kt)("em",{parentName:"p"},"--alias"),"\nduring installation, or ",(0,i.kt)("em",{parentName:"p"},"rename")," it (explained later)."),(0,i.kt)("p",null,"The following is an example when running Optimus (without any command):"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"...\nAvailable Commands:\n ...\n extension Operate with extension\n ...\n valor Execute gojek/optimus-extension-valor [v0.0.4] extension\n version Print the client version information\n...\n")),(0,i.kt)("h3",{id:"executing"},"Executing"),(0,i.kt)("p",null,"In order to execute an extension, make sure to follow the installation process described above. After installation\nis finished, simply run the extension with the following command:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus [extension name or alias]\n")),(0,i.kt)("p",null,"Example of valor:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus valor\n")),(0,i.kt)("p",null,"Operation\nThe user can do some operations to an extension. This section explains more about the available commands.\nDo note that these commands are available on the installed extensions. For more detail, run the following command:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension [extension name or alias]\n")),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor\n\nThe above command shows all available commands for valor extension.\n\nOutput:\n\nSub-command to operate over extension [gojek/optimus-extension-valor@v0.0.4]\n\nUSAGE\n optimus extension valor [flags]\n\nCORE COMMANDS\n activate activate is a sub command to allow user to activate an installed tag\n describe describe is a sub command to allow user to describe extension\n rename rename is a sub command to allow user to rename an extension command\n uninstall uninstall is a sub command to allow user to uninstall a specified tag of an extension\n upgrade upgrade is a sub command to allow user to upgrade an extension command\n\nINHERITED FLAGS\n --help Show help for command\n --no-color Disable colored output\n -v, --verbose if true, then more message will be provided if error encountered\n")),(0,i.kt)("h3",{id:"activate"},"Activate"),(0,i.kt)("p",null,"Activate a specific tag when running extension. For example, if the user has two versions of valor,\nwhich is v0.0.1 and v0.0.2, then by specifying the correct tag, the user can just switch between tag."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor activate v0.0.1\n")),(0,i.kt)("h3",{id:"describe"},"Describe"),(0,i.kt)("p",null,"Describes general information about an extension, such information includes all available releases of an extension\nin the local, which release is active, and more."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor describe\n")),(0,i.kt)("h3",{id:"rename"},"Rename"),(0,i.kt)("p",null,"Rename a specific extension to another command that is not reserved. By default, Optimus tries to figure out the\nappropriate command name from its project name. However, sometimes the extension name is not convenient like it\nbeing too long or the user just wants to change it."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor rename vl\n")),(0,i.kt)("h3",{id:"uninstall"},"Uninstall"),(0,i.kt)("p",null,"Uninstalls extension as a whole or only a specific tag. This allows the user to do some cleanup to preserve some\nstorage or to resolve some issues. By default, Optimus will uninstall the extension as a whole. To target a specific tag,\nuse the flag --tag."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor uninstall\n")),(0,i.kt)("h3",{id:"upgrade"},"Upgrade"),(0,i.kt)("p",null,"Upgrade allows the user to upgrade a certain extension to its latest tag. Although the user can use the install command,\nusing this command is shorter and easier as the user only needs to specify the installed extension."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor upgrade\n")))}c.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[4648],{3905:(e,t,n)=>{n.d(t,{Zo:()=>m,kt:()=>h});var a=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var r=a.createContext({}),u=function(e){var t=a.useContext(r),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},m=function(e){var t=u(e.components);return a.createElement(r.Provider,{value:t},e.children)},p="mdxType",c={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,r=e.parentName,m=s(e,["components","mdxType","originalType","parentName"]),p=u(n),d=i,h=p["".concat(r,".").concat(d)]||p[d]||c[d]||o;return n?a.createElement(h,l(l({ref:t},m),{},{components:n})):a.createElement(h,l({ref:t},m))}));function h(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,l=new Array(o);l[0]=d;var s={};for(var r in t)hasOwnProperty.call(t,r)&&(s[r]=t[r]);s.originalType=e,s[p]="string"==typeof e?e:i,l[1]=s;for(var u=2;u{n.r(t),n.d(t,{assets:()=>r,contentTitle:()=>l,default:()=>c,frontMatter:()=>o,metadata:()=>s,toc:()=>u});var a=n(7462),i=(n(7294),n(3905));const o={},l="Work with Extension",s={unversionedId:"client-guide/work-with-extension",id:"client-guide/work-with-extension",title:"Work with Extension",description:"Extension helps you to include third-party or arbitrary implementation as part of Optimus. Currently, the extension is",source:"@site/docs/client-guide/work-with-extension.md",sourceDirName:"client-guide",slug:"/client-guide/work-with-extension",permalink:"/optimus/docs/client-guide/work-with-extension",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/work-with-extension.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Replay a Job (Backfill)",permalink:"/optimus/docs/client-guide/replay-a-job"},next:{title:"Defining Scheduler Version",permalink:"/optimus/docs/client-guide/defining-scheduler-version"}},r={},u=[{value:"Warning",id:"warning",level:2},{value:"Limitation",id:"limitation",level:2},{value:"Creating",id:"creating",level:2},{value:"Commands",id:"commands",level:2},{value:"Installation",id:"installation",level:3},{value:"Executing",id:"executing",level:3},{value:"Activate",id:"activate",level:3},{value:"Describe",id:"describe",level:3},{value:"Rename",id:"rename",level:3},{value:"Uninstall",id:"uninstall",level:3},{value:"Upgrade",id:"upgrade",level:3}],m={toc:u},p="wrapper";function c(e){let{components:t,...n}=e;return(0,i.kt)(p,(0,a.Z)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"work-with-extension"},"Work with Extension"),(0,i.kt)("p",null,"Extension helps you to include third-party or arbitrary implementation as part of Optimus. Currently, the extension is\ndesigned for when the user is running it as CLI."),(0,i.kt)("h2",{id:"warning"},"Warning"),(0,i.kt)("p",null,"Extension is basically an executable file outside Optimus. ",(0,i.kt)("em",{parentName:"p"},"We do not guarantee whether an extension is safe or not"),".\nWe suggest checking the extension itself, whether it is safe to run in your local or not, before installing and running it."),(0,i.kt)("h2",{id:"limitation"},"Limitation"),(0,i.kt)("p",null,"Extension is designed to be similar to ",(0,i.kt)("a",{parentName:"p",href:"https://cli.github.com/manual/gh_extension"},"GitHub extension"),". However, since\nit's still in the early stage, some limitations are there."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"extension is only an executable file"),(0,i.kt)("li",{parentName:"ul"},"installation only looks at the GitHub asset according to the running system OS and Architecture"),(0,i.kt)("li",{parentName:"ul"},"convention for extension:",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"extension repository should follow optimus-extension-","[name of extension]"," (example: ",(0,i.kt)("a",{parentName:"li",href:"https://github.com/gojek/optimus-extension-valor"},"optimus-extension-valor"),")"),(0,i.kt)("li",{parentName:"ul"},"asset being considered is binary with suffix ...","[OS]","-","[ARC]"," (example: when installing valor, if the user's OS is "),(0,i.kt)("li",{parentName:"ul"},"Linux and the architecture is AMD64, then installation will consider valor_linux-amd64 as binary to be executed)")))),(0,i.kt)("h2",{id:"creating"},"Creating"),(0,i.kt)("p",null,"Extension is designed to be open. Anyone could create their own extension. And as long as it is available,\nanyone could install it. In order to create it, the following are the basic steps to do:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Decide the name of the extension, example: ",(0,i.kt)("em",{parentName:"li"},"valor")),(0,i.kt)("li",{parentName:"ul"},"Create a GitHub repository that follows the convention, example: ",(0,i.kt)("em",{parentName:"li"},"optimus-extension-valor")),(0,i.kt)("li",{parentName:"ul"},"Put some implementation and asset with the name based on the convention, example: ",(0,i.kt)("em",{parentName:"li"},"valor_linux-amd64, valor_darwin-amd64"),", and more."),(0,i.kt)("li",{parentName:"ul"},"Ensure it is available for anyone to download")),(0,i.kt)("h2",{id:"commands"},"Commands"),(0,i.kt)("p",null,"Optimus supports some commands to help operate on extension."),(0,i.kt)("h3",{id:"installation"},"Installation"),(0,i.kt)("p",null,"You can run the installation using Optimus sub-command install under the extension. In order to install an extension, run the following command:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension install REMOTE [flags]\n")),(0,i.kt)("p",null,"You can use the ",(0,i.kt)("em",{parentName:"p"},"--alias")," flag to change the command name, since by default, Optimus will try to figure it out by itself.\nAlthough, during this process, sometimes an extension name conflicts with the reserved commands. This flag helps to\nresolve that. But, do note that this flag cannot be used to rename an ",(0,i.kt)("em",{parentName:"p"},"installed")," extension. To do such a thing, check rename."),(0,i.kt)("p",null,"REMOTE is the Github remote path where to look for the extension. REMOTE can be in the form of"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"OWNER/PROJECT"),(0,i.kt)("li",{parentName:"ul"},"github.com/OWNER/PROJECT"),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("a",{parentName:"li",href:"https://www.github.com/OWNER/PROJECT"},"https://www.github.com/OWNER/PROJECT"))),(0,i.kt)("p",null,"One example of such an extension is Valor. So, going back to the example above, installing it is like this:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension install gojek/optimus-extension-valor@v0.0.4\n")),(0,i.kt)("p",null,"or"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension install github.com/gojek/optimus-extension-valor@v0.0.4\n")),(0,i.kt)("p",null,"or"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension install https://github.com/gojek/optimus-extension-valor@v0.0.4\n")),(0,i.kt)("p",null,"Installation process is then in progress. If the installation is a success, the user can show it by running:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus --help\n")),(0,i.kt)("p",null,"A new command named after the extension will be available. For example, if the extension name is ",(0,i.kt)("em",{parentName:"p"},"optimus-extension-valor"),",\nthen by default the command named valor will be available. If the user wishes to change it, they can use ",(0,i.kt)("em",{parentName:"p"},"--alias"),"\nduring installation, or ",(0,i.kt)("em",{parentName:"p"},"rename")," it (explained later)."),(0,i.kt)("p",null,"The following is an example when running Optimus (without any command):"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"...\nAvailable Commands:\n ...\n extension Operate with extension\n ...\n valor Execute gojek/optimus-extension-valor [v0.0.4] extension\n version Print the client version information\n...\n")),(0,i.kt)("h3",{id:"executing"},"Executing"),(0,i.kt)("p",null,"In order to execute an extension, make sure to follow the installation process described above. After installation\nis finished, simply run the extension with the following command:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus [extension name or alias]\n")),(0,i.kt)("p",null,"Example of valor:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus valor\n")),(0,i.kt)("p",null,"Operation\nThe user can do some operations to an extension. This section explains more about the available commands.\nDo note that these commands are available on the installed extensions. For more detail, run the following command:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension [extension name or alias]\n")),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor\n\nThe above command shows all available commands for valor extension.\n\nOutput:\n\nSub-command to operate over extension [gojek/optimus-extension-valor@v0.0.4]\n\nUSAGE\n optimus extension valor [flags]\n\nCORE COMMANDS\n activate activate is a sub command to allow user to activate an installed tag\n describe describe is a sub command to allow user to describe extension\n rename rename is a sub command to allow user to rename an extension command\n uninstall uninstall is a sub command to allow user to uninstall a specified tag of an extension\n upgrade upgrade is a sub command to allow user to upgrade an extension command\n\nINHERITED FLAGS\n --help Show help for command\n --no-color Disable colored output\n -v, --verbose if true, then more message will be provided if error encountered\n")),(0,i.kt)("h3",{id:"activate"},"Activate"),(0,i.kt)("p",null,"Activate a specific tag when running extension. For example, if the user has two versions of valor,\nwhich is v0.0.1 and v0.0.2, then by specifying the correct tag, the user can just switch between tag."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor activate v0.0.1\n")),(0,i.kt)("h3",{id:"describe"},"Describe"),(0,i.kt)("p",null,"Describes general information about an extension, such information includes all available releases of an extension\nin the local, which release is active, and more."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor describe\n")),(0,i.kt)("h3",{id:"rename"},"Rename"),(0,i.kt)("p",null,"Rename a specific extension to another command that is not reserved. By default, Optimus tries to figure out the\nappropriate command name from its project name. However, sometimes the extension name is not convenient like it\nbeing too long or the user just wants to change it."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor rename vl\n")),(0,i.kt)("h3",{id:"uninstall"},"Uninstall"),(0,i.kt)("p",null,"Uninstalls extension as a whole or only a specific tag. This allows the user to do some cleanup to preserve some\nstorage or to resolve some issues. By default, Optimus will uninstall the extension as a whole. To target a specific tag,\nuse the flag --tag."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor uninstall\n")),(0,i.kt)("h3",{id:"upgrade"},"Upgrade"),(0,i.kt)("p",null,"Upgrade allows the user to upgrade a certain extension to its latest tag. Although the user can use the install command,\nusing this command is shorter and easier as the user only needs to specify the installed extension."),(0,i.kt)("p",null,"Example:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus extension valor upgrade\n")))}c.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/65608a03.ac421215.js b/assets/js/65608a03.b6ae1562.js similarity index 98% rename from assets/js/65608a03.ac421215.js rename to assets/js/65608a03.b6ae1562.js index 15e01610fb..60ecda2136 100644 --- a/assets/js/65608a03.ac421215.js +++ b/assets/js/65608a03.b6ae1562.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[7139],{3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>m});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var c=n.createContext({}),l=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},p=function(e){var t=l(e.components);return n.createElement(c.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},f=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,c=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(r),f=a,m=u["".concat(c,".").concat(f)]||u[f]||d[f]||o;return r?n.createElement(m,i(i({ref:t},p),{},{components:r})):n.createElement(m,i({ref:t},p))}));function m(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=f;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s[u]="string"==typeof e?e:a,i[1]=s;for(var l=2;l{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>o,metadata:()=>s,toc:()=>l});var n=r(7462),a=(r(7294),r(3905));const o={},i="FAQ",s={unversionedId:"reference/faq",id:"reference/faq",title:"FAQ",description:"- I want to run a DDL/DML query like DELETE, how can I do that?",source:"@site/docs/reference/faq.md",sourceDirName:"reference",slug:"/reference/faq",permalink:"/optimus/docs/reference/faq",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/reference/faq.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Metrics",permalink:"/optimus/docs/reference/metrics"}},c={},l=[],p={toc:l},u="wrapper";function d(e){let{components:t,...r}=e;return(0,a.kt)(u,(0,n.Z)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"faq"},"FAQ"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"I want to run a DDL/DML query like DELETE, how can I do that?")),(0,a.kt)("p",{parentName:"li"},"Write SQL query as you would write in BQ UI and select the load method as MERGE.\nThis will execute even if the query does not contain a merge statement.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"What should not be changed once the specifications are created?")),(0,a.kt)("p",{parentName:"li"},"Optimus uses Airflow for scheduling job execution and it does not support change of start_date & schedule_interval\nonce the dag is created. For that please delete the existing one or recreate it with a different suffix.\nAlso make sure you don\u2019t change the dag name if you don\u2019t want to lose the run history of Airflow.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"Can I have a job with only a transformation task (without a hook) or the other way around?")),(0,a.kt)("p",{parentName:"li"},"Transformation task is mandatory but hook is optional. You can have a transformation task without a hook but cannot\nhave a hook without a transformation task.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"I have a job with a modified view source, however, it does not detect as modified when running the job replace-all command.\nHow should I apply the change?")),(0,a.kt)("p",{parentName:"li"},"It does not detect as modified as the specification and the assets of the job itself is not changed. Do run the job refresh command.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"My job is failing due to the resource is not sufficient. Can I scale a specific job to have a bigger CPU / memory limit?")),(0,a.kt)("p",{parentName:"li"},"Yes, resource requests and limits are configurable in the job specification\u2019s metadata field.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"I removed a resource specification, but why does the resource is not deleted in the BigQuery project?")),(0,a.kt)("p",{parentName:"li"},"Optimus currently does not support resource deletion to avoid any accidental deletion.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"I removed a job specification, but why does it still appear in Airflow UI?")),(0,a.kt)("p",{parentName:"li"},"You might want to check the log of the replace-all command you are using. It will show whether your job has been\nsuccessfully deleted or not. If not, the possible causes are the job is being used by another job as a dependency.\nYou can force delete it through API if needed."),(0,a.kt)("p",{parentName:"li"},"If the job has been successfully deleted, it might take time for the deletion to reflect which depends on your Airflow sync or DAG load configuration."))))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[7139],{3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>m});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var c=n.createContext({}),l=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},p=function(e){var t=l(e.components);return n.createElement(c.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},f=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,c=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=l(r),f=a,m=u["".concat(c,".").concat(f)]||u[f]||d[f]||o;return r?n.createElement(m,i(i({ref:t},p),{},{components:r})):n.createElement(m,i({ref:t},p))}));function m(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=f;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s[u]="string"==typeof e?e:a,i[1]=s;for(var l=2;l{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>o,metadata:()=>s,toc:()=>l});var n=r(7462),a=(r(7294),r(3905));const o={},i="FAQ",s={unversionedId:"reference/faq",id:"reference/faq",title:"FAQ",description:"- I want to run a DDL/DML query like DELETE, how can I do that?",source:"@site/docs/reference/faq.md",sourceDirName:"reference",slug:"/reference/faq",permalink:"/optimus/docs/reference/faq",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/reference/faq.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Metrics",permalink:"/optimus/docs/reference/metrics"}},c={},l=[],p={toc:l},u="wrapper";function d(e){let{components:t,...r}=e;return(0,a.kt)(u,(0,n.Z)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"faq"},"FAQ"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"I want to run a DDL/DML query like DELETE, how can I do that?")),(0,a.kt)("p",{parentName:"li"},"Write SQL query as you would write in BQ UI and select the load method as MERGE.\nThis will execute even if the query does not contain a merge statement.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"What should not be changed once the specifications are created?")),(0,a.kt)("p",{parentName:"li"},"Optimus uses Airflow for scheduling job execution and it does not support change of start_date & schedule_interval\nonce the dag is created. For that please delete the existing one or recreate it with a different suffix.\nAlso make sure you don\u2019t change the dag name if you don\u2019t want to lose the run history of Airflow.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"Can I have a job with only a transformation task (without a hook) or the other way around?")),(0,a.kt)("p",{parentName:"li"},"Transformation task is mandatory but hook is optional. You can have a transformation task without a hook but cannot\nhave a hook without a transformation task.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"I have a job with a modified view source, however, it does not detect as modified when running the job replace-all command.\nHow should I apply the change?")),(0,a.kt)("p",{parentName:"li"},"It does not detect as modified as the specification and the assets of the job itself is not changed. Do run the job refresh command.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"My job is failing due to the resource is not sufficient. Can I scale a specific job to have a bigger CPU / memory limit?")),(0,a.kt)("p",{parentName:"li"},"Yes, resource requests and limits are configurable in the job specification\u2019s metadata field.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"I removed a resource specification, but why does the resource is not deleted in the BigQuery project?")),(0,a.kt)("p",{parentName:"li"},"Optimus currently does not support resource deletion to avoid any accidental deletion.")),(0,a.kt)("li",{parentName:"ul"},(0,a.kt)("p",{parentName:"li"},(0,a.kt)("strong",{parentName:"p"},"I removed a job specification, but why does it still appear in Airflow UI?")),(0,a.kt)("p",{parentName:"li"},"You might want to check the log of the replace-all command you are using. It will show whether your job has been\nsuccessfully deleted or not. If not, the possible causes are the job is being used by another job as a dependency.\nYou can force delete it through API if needed."),(0,a.kt)("p",{parentName:"li"},"If the job has been successfully deleted, it might take time for the deletion to reflect which depends on your Airflow sync or DAG load configuration."))))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/6ae6f4b6.23b2ede1.js b/assets/js/6ae6f4b6.15eed989.js similarity index 98% rename from assets/js/6ae6f4b6.23b2ede1.js rename to assets/js/6ae6f4b6.15eed989.js index 9d8bf1327d..471182ec09 100644 --- a/assets/js/6ae6f4b6.23b2ede1.js +++ b/assets/js/6ae6f4b6.15eed989.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6986],{3905:(e,t,n)=>{n.d(t,{Zo:()=>d,kt:()=>y});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},d=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},l="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,s=e.parentName,d=i(e,["components","mdxType","originalType","parentName"]),l=p(n),m=o,y=l["".concat(s,".").concat(m)]||l[m]||u[m]||a;return n?r.createElement(y,c(c({ref:t},d),{},{components:n})):r.createElement(y,c({ref:t},d))}));function y(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,c=new Array(a);c[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i[l]="string"==typeof e?e:o,c[1]=i;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>c,default:()=>u,frontMatter:()=>a,metadata:()=>i,toc:()=>p});var r=n(7462),o=(n(7294),n(3905));const a={},c="Dependency",i={unversionedId:"concepts/dependency",id:"concepts/dependency",title:"Dependency",description:"A job can have a source and a destination to start with. This source could be a resource managed by Optimus or",source:"@site/docs/concepts/dependency.md",sourceDirName:"concepts",slug:"/concepts/dependency",permalink:"/optimus/docs/concepts/dependency",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/dependency.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Job Run",permalink:"/optimus/docs/concepts/job-run"},next:{title:"Macros",permalink:"/optimus/docs/concepts/macros"}},s={},p=[],d={toc:p},l="wrapper";function u(e){let{components:t,...n}=e;return(0,o.kt)(l,(0,r.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"dependency"},"Dependency"),(0,o.kt)("p",null,"A job can have a source and a destination to start with. This source could be a resource managed by Optimus or\nnon-managed like an S3 bucket. If the dependency is managed by Optimus, it is obvious that in an ETL pipeline, it is\nrequired for the dependency to finish successfully first before the dependent job can start."),(0,o.kt)("p",null,"There are 2 types of dependency depending on how to configure it:"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Type"),(0,o.kt)("th",{parentName:"tr",align:null},"Description"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Inferred"),(0,o.kt)("td",{parentName:"tr",align:null},"Automatically detected through assets. The logic on how to detect the dependency is configured in each of the ",(0,o.kt)("a",{parentName:"td",href:"/optimus/docs/concepts/plugin"},"plugins"),".")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Static"),(0,o.kt)("td",{parentName:"tr",align:null},"Configured through job.yaml")))),(0,o.kt)("p",null,"Optimus also supports job dependency to cross-optimus servers. These Optimus servers are considered external resource\nmanagers, where Optimus will look for the job sources that have not been resolved internally and create the dependency.\nThese resource managers should be configured in the server configuration."))}u.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6986],{3905:(e,t,n)=>{n.d(t,{Zo:()=>d,kt:()=>y});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},d=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},l="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,s=e.parentName,d=i(e,["components","mdxType","originalType","parentName"]),l=p(n),m=o,y=l["".concat(s,".").concat(m)]||l[m]||u[m]||a;return n?r.createElement(y,c(c({ref:t},d),{},{components:n})):r.createElement(y,c({ref:t},d))}));function y(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,c=new Array(a);c[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i[l]="string"==typeof e?e:o,c[1]=i;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>c,default:()=>u,frontMatter:()=>a,metadata:()=>i,toc:()=>p});var r=n(7462),o=(n(7294),n(3905));const a={},c="Dependency",i={unversionedId:"concepts/dependency",id:"concepts/dependency",title:"Dependency",description:"A job can have a source and a destination to start with. This source could be a resource managed by Optimus or",source:"@site/docs/concepts/dependency.md",sourceDirName:"concepts",slug:"/concepts/dependency",permalink:"/optimus/docs/concepts/dependency",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/dependency.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Job Run",permalink:"/optimus/docs/concepts/job-run"},next:{title:"Macros",permalink:"/optimus/docs/concepts/macros"}},s={},p=[],d={toc:p},l="wrapper";function u(e){let{components:t,...n}=e;return(0,o.kt)(l,(0,r.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"dependency"},"Dependency"),(0,o.kt)("p",null,"A job can have a source and a destination to start with. This source could be a resource managed by Optimus or\nnon-managed like an S3 bucket. If the dependency is managed by Optimus, it is obvious that in an ETL pipeline, it is\nrequired for the dependency to finish successfully first before the dependent job can start."),(0,o.kt)("p",null,"There are 2 types of dependency depending on how to configure it:"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Type"),(0,o.kt)("th",{parentName:"tr",align:null},"Description"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Inferred"),(0,o.kt)("td",{parentName:"tr",align:null},"Automatically detected through assets. The logic on how to detect the dependency is configured in each of the ",(0,o.kt)("a",{parentName:"td",href:"/optimus/docs/concepts/plugin"},"plugins"),".")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Static"),(0,o.kt)("td",{parentName:"tr",align:null},"Configured through job.yaml")))),(0,o.kt)("p",null,"Optimus also supports job dependency to cross-optimus servers. These Optimus servers are considered external resource\nmanagers, where Optimus will look for the job sources that have not been resolved internally and create the dependency.\nThese resource managers should be configured in the server configuration."))}u.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/6e97b392.d9ae6e69.js b/assets/js/6e97b392.e912cd31.js similarity index 98% rename from assets/js/6e97b392.d9ae6e69.js rename to assets/js/6e97b392.e912cd31.js index 03044e15c1..ee21cf14fd 100644 --- a/assets/js/6e97b392.d9ae6e69.js +++ b/assets/js/6e97b392.e912cd31.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3390],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>m});var r=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var a=r.createContext({}),u=function(e){var t=r.useContext(a),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},p=function(e){var t=u(e.components);return r.createElement(a.Provider,{value:t},e.children)},c="mdxType",g={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},d=r.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,a=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),c=u(n),d=i,m=c["".concat(a,".").concat(d)]||c[d]||g[d]||o;return n?r.createElement(m,s(s({ref:t},p),{},{components:n})):r.createElement(m,s({ref:t},p))}));function m(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=d;var l={};for(var a in t)hasOwnProperty.call(t,a)&&(l[a]=t[a]);l.originalType=e,l[c]="string"==typeof e?e:i,s[1]=l;for(var u=2;u{n.r(t),n.d(t,{assets:()=>a,contentTitle:()=>s,default:()=>g,frontMatter:()=>o,metadata:()=>l,toc:()=>u});var r=n(7462),i=(n(7294),n(3905));const o={},s="Installing Plugins",l={unversionedId:"server-guide/installing-plugins",id:"server-guide/installing-plugins",title:"Installing Plugins",description:"Plugin needs to be installed in the Optimus server before it can be used. Optimus uses the following directories for",source:"@site/docs/server-guide/installing-plugins.md",sourceDirName:"server-guide",slug:"/server-guide/installing-plugins",permalink:"/optimus/docs/server-guide/installing-plugins",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/server-guide/installing-plugins.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Server Configuration",permalink:"/optimus/docs/server-guide/configuration"},next:{title:"Starting Optimus Server",permalink:"/optimus/docs/server-guide/starting-optimus-server"}},a={},u=[],p={toc:u},c="wrapper";function g(e){let{components:t,...n}=e;return(0,i.kt)(c,(0,r.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"installing-plugins"},"Installing Plugins"),(0,i.kt)("p",null,"Plugin needs to be installed in the Optimus server before it can be used. Optimus uses the following directories for\ndiscovering plugin binaries"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"./.plugins\n./\n/\n/.optimus/plugins\n$HOME/.optimus/plugins\n/usr/bin\n/usr/local/bin\n")),(0,i.kt)("p",null,"Even though the above list of directories is involved in plugin discovery, it is advised to use .plugins in the\ncurrent working directory of the project or Optimus binary."),(0,i.kt)("p",null,"To simplify installation, you can add plugin artifacts in the server config:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},"plugin:\n artifacts:\n - https://...path/to/optimus-plugin-neo.yaml # http\n - http://.../plugins.zip # zip\n - ../transformers/optimus-bq2bq_darwin_arm64 # relative paths\n - ../transformers/optimus-plugin-neo.yaml\n")),(0,i.kt)("p",null,"Run below command to auto-install the plugins in the ",(0,i.kt)("inlineCode",{parentName:"p"},".plugins")," directory."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus plugin install -c config.yaml # This will install plugins in the `.plugins` folder.\n")))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3390],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>m});var r=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var a=r.createContext({}),u=function(e){var t=r.useContext(a),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},p=function(e){var t=u(e.components);return r.createElement(a.Provider,{value:t},e.children)},c="mdxType",g={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},d=r.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,a=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),c=u(n),d=i,m=c["".concat(a,".").concat(d)]||c[d]||g[d]||o;return n?r.createElement(m,s(s({ref:t},p),{},{components:n})):r.createElement(m,s({ref:t},p))}));function m(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,s=new Array(o);s[0]=d;var l={};for(var a in t)hasOwnProperty.call(t,a)&&(l[a]=t[a]);l.originalType=e,l[c]="string"==typeof e?e:i,s[1]=l;for(var u=2;u{n.r(t),n.d(t,{assets:()=>a,contentTitle:()=>s,default:()=>g,frontMatter:()=>o,metadata:()=>l,toc:()=>u});var r=n(7462),i=(n(7294),n(3905));const o={},s="Installing Plugins",l={unversionedId:"server-guide/installing-plugins",id:"server-guide/installing-plugins",title:"Installing Plugins",description:"Plugin needs to be installed in the Optimus server before it can be used. Optimus uses the following directories for",source:"@site/docs/server-guide/installing-plugins.md",sourceDirName:"server-guide",slug:"/server-guide/installing-plugins",permalink:"/optimus/docs/server-guide/installing-plugins",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/server-guide/installing-plugins.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Server Configuration",permalink:"/optimus/docs/server-guide/configuration"},next:{title:"Starting Optimus Server",permalink:"/optimus/docs/server-guide/starting-optimus-server"}},a={},u=[],p={toc:u},c="wrapper";function g(e){let{components:t,...n}=e;return(0,i.kt)(c,(0,r.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"installing-plugins"},"Installing Plugins"),(0,i.kt)("p",null,"Plugin needs to be installed in the Optimus server before it can be used. Optimus uses the following directories for\ndiscovering plugin binaries"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"./.plugins\n./\n/\n/.optimus/plugins\n$HOME/.optimus/plugins\n/usr/bin\n/usr/local/bin\n")),(0,i.kt)("p",null,"Even though the above list of directories is involved in plugin discovery, it is advised to use .plugins in the\ncurrent working directory of the project or Optimus binary."),(0,i.kt)("p",null,"To simplify installation, you can add plugin artifacts in the server config:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},"plugin:\n artifacts:\n - https://...path/to/optimus-plugin-neo.yaml # http\n - http://.../plugins.zip # zip\n - ../transformers/optimus-bq2bq_darwin_arm64 # relative paths\n - ../transformers/optimus-plugin-neo.yaml\n")),(0,i.kt)("p",null,"Run below command to auto-install the plugins in the ",(0,i.kt)("inlineCode",{parentName:"p"},".plugins")," directory."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus plugin install -c config.yaml # This will install plugins in the `.plugins` folder.\n")))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/79c9e2d7.b6864922.js b/assets/js/79c9e2d7.4b681d2a.js similarity index 99% rename from assets/js/79c9e2d7.b6864922.js rename to assets/js/79c9e2d7.4b681d2a.js index 05e9c26ed1..5af94b5c54 100644 --- a/assets/js/79c9e2d7.b6864922.js +++ b/assets/js/79c9e2d7.4b681d2a.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6449],{3905:(e,t,n)=>{n.d(t,{Zo:()=>d,kt:()=>m});var o=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function a(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=o.createContext({}),p=function(e){var t=o.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},d=function(e){var t=p(e.components);return o.createElement(s.Provider,{value:t},e.children)},c="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},h=o.forwardRef((function(e,t){var n=e.components,i=e.mdxType,l=e.originalType,s=e.parentName,d=r(e,["components","mdxType","originalType","parentName"]),c=p(n),h=i,m=c["".concat(s,".").concat(h)]||c[h]||u[h]||l;return n?o.createElement(m,a(a({ref:t},d),{},{components:n})):o.createElement(m,a({ref:t},d))}));function m(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var l=n.length,a=new Array(l);a[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r[c]="string"==typeof e?e:i,a[1]=r;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>a,default:()=>u,frontMatter:()=>l,metadata:()=>r,toc:()=>p});var o=n(7462),i=(n(7294),n(3905));const l={},a=void 0,r={unversionedId:"rfcs/improving_time_and_flow_of_deployment",id:"rfcs/improving_time_and_flow_of_deployment",title:"improving_time_and_flow_of_deployment",description:"- Feature Name: Improve Time & Flow of the core Optimus Job Deployment, Replay, and Backup",source:"@site/docs/rfcs/20220216_improving_time_and_flow_of_deployment.md",sourceDirName:"rfcs",slug:"/rfcs/improving_time_and_flow_of_deployment",permalink:"/optimus/docs/rfcs/improving_time_and_flow_of_deployment",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220216_improving_time_and_flow_of_deployment.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220216,frontMatter:{}},s={},p=[{value:"Background :",id:"background-",level:2},{value:"Deploy",id:"deploy",level:3},{value:"Replay Request",id:"replay-request",level:3},{value:"Backup Request",id:"backup-request",level:3},{value:"Expected Behavior",id:"expected-behavior",level:2},{value:"Deploy Job",id:"deploy-job",level:3},{value:"Create Job",id:"create-job",level:3},{value:"Delete Job",id:"delete-job",level:3},{value:"Refresh Job",id:"refresh-job",level:3},{value:"Create Resource",id:"create-resource",level:3},{value:"Delete Resource",id:"delete-resource",level:3},{value:"Replay & Backup",id:"replay--backup",level:3},{value:"Approach :",id:"approach-",level:2},{value:"Checking which jobs are modified?",id:"checking-which-jobs-are-modified",level:3},{value:"Persistence",id:"persistence",level:3},{value:"Event-Based Mechanism in Deployment",id:"event-based-mechanism-in-deployment",level:3},{value:"Handling Dependency Resolution Failure",id:"handling-dependency-resolution-failure",level:3},{value:"Handling Modified View",id:"handling-modified-view",level:3},{value:"CLI Perspective",id:"cli-perspective",level:3},{value:"Other Thoughts:",id:"other-thoughts",level:2}],d={toc:p},c="wrapper";function u(e){let{components:t,...n}=e;return(0,i.kt)(c,(0,o.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Feature Name: Improve Time & Flow of the core Optimus Job Deployment, Replay, and Backup"),(0,i.kt)("li",{parentName:"ul"},"Status: Draft"),(0,i.kt)("li",{parentName:"ul"},"Start Date: 2022-02-16"),(0,i.kt)("li",{parentName:"ul"},"Authors: Arinda & Sravan")),(0,i.kt)("h1",{id:"summary"},"Summary"),(0,i.kt)("p",null,"It is observed that the deployment of a project with more than 1000 jobs took around 6 minutes to complete, and the\nreplay request for the same project took around 4 minutes. An analysis to improve the time taken for this is needed,\nespecially if the project will be broke down to multiple namespaces. This will cause problem, as 10 namespaces might\ncan consume 6 minutes * 10 times."),(0,i.kt)("h1",{id:"technical-design"},"Technical Design"),(0,i.kt)("h2",{id:"background-"},"Background :"),(0,i.kt)("p",null,"Understanding the current process of the mentioned issues:"),(0,i.kt)("h3",{id:"deploy"},"Deploy"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolving dependencies for all the jobs in the requested project"),(0,i.kt)("li",{parentName:"ul"},"Resolving priorities for all the jobs"),(0,i.kt)("li",{parentName:"ul"},"Compiling the jobs within requested namespace"),(0,i.kt)("li",{parentName:"ul"},"Uploading compiled jobs to storage")),(0,i.kt)("h3",{id:"replay-request"},"Replay Request"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolving dependencies for all the jobs in the requested project"),(0,i.kt)("li",{parentName:"ul"},"Clearing scheduler dag run(s)")),(0,i.kt)("h3",{id:"backup-request"},"Backup Request"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolving dependencies for all the jobs in the requested project"),(0,i.kt)("li",{parentName:"ul"},"Duplicate table")),(0,i.kt)("p",null,"All the processes above need dependency resolution. When resolving dependency, it is being done for ALL the jobs in the\nproject, regardless of the namespace and regardless if it has changed or not. For every job (bq2bq for example), Optimus\nwill call each of the jobs the GenerateDependencies function in the plugin, and do a dry run hitting the Bigquery API.\nThis process has been done in parallel."),(0,i.kt)("p",null,"To simulate, let\u2019s say there are 1000 bq2bq jobs in a project."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"}," Job | Upstream\n--------------------------\n A | -\n B | A\n C | A, B\n D | C \n ... | ...\n")),(0,i.kt)("p",null,"There is a change in job C, that it no longer has dependency to job A. When it happens, when deploying, currently\nOptimus will resolve dependencies for all 1000 jobs. While in fact, the changed dependencies will only happen for job C.\nThere is only a slight difference where upstream is no longer A and B but only B."),(0,i.kt)("p",null,"Currently, Optimus is resolving dependencies every time it is needed because it is not stored anywhere, due to keep\nchanging dependencies. However, now we are seeing a timing problem, and the fact that not all jobs need to be dependency\nresolved everytime there\u2019s a deployment, a modification can be considered."),(0,i.kt)("p",null,"As part of this issue, we are also revisiting the current flow of job deployment process."),(0,i.kt)("h2",{id:"expected-behavior"},"Expected Behavior"),(0,i.kt)("h3",{id:"deploy-job"},"Deploy Job"),(0,i.kt)("p",null,"Accepts the whole state of the namespace/project. What is not available will be deleted."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Identify the added / modified / deleted jobs"),(0,i.kt)("li",{parentName:"ul"},"Resolve dependency only for the added or modified jobs and persist the dependency to DB"),(0,i.kt)("li",{parentName:"ul"},"Do priority resolution for all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Compile all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Upload the compiled jobs")),(0,i.kt)("p",null,"The difference between the expected behavior and current implementation is that we will only resolve dependency for\nwhat are necessary, and we will compile all the jobs in the project regardless the namespace. Compile and deploy all\njobs in the project is necessary to avoid below use case:"),(0,i.kt)("p",null,"Let's say in a single project, lies these 4 jobs. Job C depend on Job B, job B depend on Job A, and Job A and Job D are\nindependent. Notice that Job C is in a different namespace."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"Job A (Namespace 1) : weight 100\n|-- Job B (Namespace 1) : weight 90\n| |-- Job C (Namespace 2) : weight 80\n|-- Job D (Namespace 1) : weight 100\n")),(0,i.kt)("p",null,"Now let's say Job E (Namespace 1) is introduced and Job B is no longer depend directly on Job A, but instead to the new\nJob E."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"Job A (Namespace 1) : weight 100\n|-- Job E (Namespace 1) : weight 90\n| |-- Job B (Namespace 1) : weight 80\n| |-- Job C (Namespace 2) : weight 70\n|-- Job D (Namespace 1) : weight 100\n")),(0,i.kt)("p",null,"Notice that Job C priority weight has been changed. This example shows that even though the changes are in Namespace 1,\nthe other namespace is also affected and needs to be recompiled and deployed."),(0,i.kt)("h3",{id:"create-job"},"Create Job"),(0,i.kt)("p",null,"Accept a single/multiple jobs to be created and deployed."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolve dependency for the requested job and persist the dependency to DB"),(0,i.kt)("li",{parentName:"ul"},"Do priority resolution for all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Compile all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Upload the compiled jobs")),(0,i.kt)("h3",{id:"delete-job"},"Delete Job"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Identify any dependent jobs using dependencies table"),(0,i.kt)("li",{parentName:"ul"},"Delete only if there are no dependencies\nTBD: Doing soft delete or move the deleted jobs to a different table")),(0,i.kt)("h3",{id:"refresh-job"},"Refresh Job"),(0,i.kt)("p",null,"Using current state of job that has been stored, redo the dependency resolution, recompile, redeploy.\nCan be useful to do clean deploy or upgrading jobs."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolve dependency for all jobs in the namespace/project and persist to DB"),(0,i.kt)("li",{parentName:"ul"},"Do priority resolution for all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Compile all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Upload the compiled jobs")),(0,i.kt)("h3",{id:"create-resource"},"Create Resource"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Deploy the requested resource"),(0,i.kt)("li",{parentName:"ul"},"Identify jobs that are dependent to the resource"),(0,i.kt)("li",{parentName:"ul"},"Resolve dependency for the jobs found"),(0,i.kt)("li",{parentName:"ul"},"Compile all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Upload the compiled jobs\nAn explanation of this behaviour can be found in ",(0,i.kt)("inlineCode",{parentName:"li"},"Handling Modified View")," section")),(0,i.kt)("h3",{id:"delete-resource"},"Delete Resource"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Identify jobs that are dependent to the requested resource"),(0,i.kt)("li",{parentName:"ul"},"Delete only if there are no dependencies")),(0,i.kt)("h3",{id:"replay--backup"},"Replay & Backup"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Get the dependencies from the dependencies table."),(0,i.kt)("li",{parentName:"ul"},"Continue to build the tree.")),(0,i.kt)("h2",{id:"approach-"},"Approach :"),(0,i.kt)("h3",{id:"checking-which-jobs-are-modified"},"Checking which jobs are modified?"),(0,i.kt)("p",null,"Currently, Optimus receives all the jobs to be deployed, compares which one to be deleted and which one to keep,\nresolves and compiles them all. Optimus does not know the state of which changed."),(0,i.kt)("p",null,"One of the possibilities is by using Job hash. Fetch the jobs from DB, hash and compare with the one requested."),(0,i.kt)("h3",{id:"persistence"},"Persistence"),(0,i.kt)("p",null,"The process can be optimized only if the dependencies are stored, so no need to resolve it all every time it is needed.\nCurrently, this is the struct of JobSpec in Optimus:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-go"},"type JobSpec struct {\n ID uuid.UUID\n Name string\n Dependencies map[string]JobSpecDependency\n ....\n}\n\ntype JobSpecDependency struct {\n Project *ProjectSpec\n Job *JobSpec\n Type JobSpecDependencyType\n}\n")),(0,i.kt)("p",null,"The Dependencies field will be filled with inferred dependency after dependency resolution is finished."),(0,i.kt)("p",null,"We can have a new table to persist the job ID dependency."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"}," job_id | UUID\n job_dependency_id | UUID\n")),(0,i.kt)("p",null,"Example"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"}," Job | Upstream\n---------------------------\n A | -\n B | A\n C | A\n C | B\n D | C\n ... | ...\n")),(0,i.kt)("p",null,"If now C has been modified to have upstream of only B, means:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Record with jobID C will be deleted"),(0,i.kt)("li",{parentName:"ul"},"Insert 1 new record: C with dependency B")),(0,i.kt)("p",null,"Advantages:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Data is available even though there are pod restarts."),(0,i.kt)("li",{parentName:"ul"},"Better visibility of current dependencies.")),(0,i.kt)("p",null,"Disadvantages:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Additional time to write/read from DB")),(0,i.kt)("h3",{id:"event-based-mechanism-in-deployment"},"Event-Based Mechanism in Deployment"),(0,i.kt)("p",null,"Revisiting the process of deployment:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"Step | Deploy | Create Job | Refresh\n---------------------------------------------------------------\nResolve dependency | Diff | Requested | All\nResolve priority | All | All | All\nCompile | All | All | All\nUpload | All | All | All\n")),(0,i.kt)("p",null,"Notice that priority resolution, compilation, and upload compiled jobs needs to be done for all the jobs in the project\nfor all the namespaces. Each of the request can be done multiple times per minute and improvisation to speed up the\nprocess is needed."),(0,i.kt)("p",null,"Whenever there is a request to do deployment, job creation, and refresh, Optimus will do dependency resolution based\non each of the cases. After it finishes, it will push an event to be picked by a worker to do priority resolution,\ncompilation, and upload asynchronously. There will be deduplication in the event coming in, to avoid doing duplicated\nprocess."),(0,i.kt)("p",null,"There will be a get deployment status API introduced to poll whether these async processes has been finished or not."),(0,i.kt)("h3",{id:"handling-dependency-resolution-failure"},"Handling Dependency Resolution Failure"),(0,i.kt)("p",null,"Currently, whenever there is a single jobs that is failing in dependency resolution, and it is within the same\nnamespace as requested, it will fail the entire process. We are avoiding the entire deployment pipeline to be blocked\nby a single job failure, but instead sending it as part of the response and proceeding the deployment until finished.\nOnly the failed jobs will not be deployed. There will be metrics being added to add more visibility around this."),(0,i.kt)("h3",{id:"handling-modified-view"},"Handling Modified View"),(0,i.kt)("p",null,"A BQ2BQ job can have a source from views. For this job, the dependency will be the underlying tables of the view. Let's\nsimulate a case where there is a change in the view source."),(0,i.kt)("p",null,"In a project, there is view ",(0,i.kt)("inlineCode",{parentName:"p"},"X")," that querying from table ",(0,i.kt)("inlineCode",{parentName:"p"},"A")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"B"),". There is also table ",(0,i.kt)("inlineCode",{parentName:"p"},"C")," that querying from View\n",(0,i.kt)("inlineCode",{parentName:"p"},"X"),". The job dependencies for this case can be summarized as:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"}," Job | Upstream\n---------------------------\n A | -\n B | -\n C | A\n C | B\n")),(0,i.kt)("p",null,"Job C has dependency to job ",(0,i.kt)("inlineCode",{parentName:"p"},"A")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"B"),", even though it is querying from view ",(0,i.kt)("inlineCode",{parentName:"p"},"X"),"."),(0,i.kt)("p",null,"Imagine a case where view ",(0,i.kt)("inlineCode",{parentName:"p"},"X")," is modified, for example no longer querying from ",(0,i.kt)("inlineCode",{parentName:"p"},"A")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"B"),", but instead only from ",(0,i.kt)("inlineCode",{parentName:"p"},"A"),".\nJob ",(0,i.kt)("inlineCode",{parentName:"p"},"C")," dependency will never be updated, since it is not considered as modified. There should be a mechanism where if\na view is updated, it will also resolve the dependency for the jobs that depend on the view."),(0,i.kt)("p",null,"To make this happen, there should be a visibility of which resources are the sources of a job, for example which job is\nusing this view as a destination and querying from this view. Optimus is a transformation tool, in the job spec we store\nwhat is the transformation destination of the job. However, we are not storing what are the sources of the transformation.\nThe only thing we have is job dependency, not resource."),(0,i.kt)("p",null,"We can add a Source URNs field to the jobs specs, or create a Job Source table. Whenever there is a change in a view\nthrough Optimus, datastore should be able to request the dependency resolution for the view's dependent and having the\ndependencies updated. We will also provide the mechanism to refresh jobs."),(0,i.kt)("h3",{id:"cli-perspective"},"CLI Perspective"),(0,i.kt)("p",null,"Deploy job per namespace (using DeployJobSpecification rpc)"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"optimus job deploy --namespace --project\n")),(0,i.kt)("p",null,"Deploy job for selected jobs (using CreateJobSpecification rpc)"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"optimus job deploy --namespace --project --jobs=(job1,job2)\n")),(0,i.kt)("p",null,"Refresh the entire namespace/project"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"optimus job refresh --namespace --project\n")),(0,i.kt)("p",null,"Refresh the selected jobs"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"optimus job refresh --namespace --project --jobs=(job1,job2)\n")),(0,i.kt)("h2",{id:"other-thoughts"},"Other Thoughts:"),(0,i.kt)("p",null,"Cache Considerations instead of persisting to PG"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Might be faster as there is no additional time for write/read from DB"),(0,i.kt)("li",{parentName:"ul"},"Data will be unavailable post pod restarts. Need to redo the dependency resolution overall"),(0,i.kt)("li",{parentName:"ul"},"Poor visibility")))}u.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6449],{3905:(e,t,n)=>{n.d(t,{Zo:()=>d,kt:()=>m});var o=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function a(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=o.createContext({}),p=function(e){var t=o.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},d=function(e){var t=p(e.components);return o.createElement(s.Provider,{value:t},e.children)},c="mdxType",u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},h=o.forwardRef((function(e,t){var n=e.components,i=e.mdxType,l=e.originalType,s=e.parentName,d=r(e,["components","mdxType","originalType","parentName"]),c=p(n),h=i,m=c["".concat(s,".").concat(h)]||c[h]||u[h]||l;return n?o.createElement(m,a(a({ref:t},d),{},{components:n})):o.createElement(m,a({ref:t},d))}));function m(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var l=n.length,a=new Array(l);a[0]=h;var r={};for(var s in t)hasOwnProperty.call(t,s)&&(r[s]=t[s]);r.originalType=e,r[c]="string"==typeof e?e:i,a[1]=r;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>a,default:()=>u,frontMatter:()=>l,metadata:()=>r,toc:()=>p});var o=n(7462),i=(n(7294),n(3905));const l={},a=void 0,r={unversionedId:"rfcs/improving_time_and_flow_of_deployment",id:"rfcs/improving_time_and_flow_of_deployment",title:"improving_time_and_flow_of_deployment",description:"- Feature Name: Improve Time & Flow of the core Optimus Job Deployment, Replay, and Backup",source:"@site/docs/rfcs/20220216_improving_time_and_flow_of_deployment.md",sourceDirName:"rfcs",slug:"/rfcs/improving_time_and_flow_of_deployment",permalink:"/optimus/docs/rfcs/improving_time_and_flow_of_deployment",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220216_improving_time_and_flow_of_deployment.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220216,frontMatter:{}},s={},p=[{value:"Background :",id:"background-",level:2},{value:"Deploy",id:"deploy",level:3},{value:"Replay Request",id:"replay-request",level:3},{value:"Backup Request",id:"backup-request",level:3},{value:"Expected Behavior",id:"expected-behavior",level:2},{value:"Deploy Job",id:"deploy-job",level:3},{value:"Create Job",id:"create-job",level:3},{value:"Delete Job",id:"delete-job",level:3},{value:"Refresh Job",id:"refresh-job",level:3},{value:"Create Resource",id:"create-resource",level:3},{value:"Delete Resource",id:"delete-resource",level:3},{value:"Replay & Backup",id:"replay--backup",level:3},{value:"Approach :",id:"approach-",level:2},{value:"Checking which jobs are modified?",id:"checking-which-jobs-are-modified",level:3},{value:"Persistence",id:"persistence",level:3},{value:"Event-Based Mechanism in Deployment",id:"event-based-mechanism-in-deployment",level:3},{value:"Handling Dependency Resolution Failure",id:"handling-dependency-resolution-failure",level:3},{value:"Handling Modified View",id:"handling-modified-view",level:3},{value:"CLI Perspective",id:"cli-perspective",level:3},{value:"Other Thoughts:",id:"other-thoughts",level:2}],d={toc:p},c="wrapper";function u(e){let{components:t,...n}=e;return(0,i.kt)(c,(0,o.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Feature Name: Improve Time & Flow of the core Optimus Job Deployment, Replay, and Backup"),(0,i.kt)("li",{parentName:"ul"},"Status: Draft"),(0,i.kt)("li",{parentName:"ul"},"Start Date: 2022-02-16"),(0,i.kt)("li",{parentName:"ul"},"Authors: Arinda & Sravan")),(0,i.kt)("h1",{id:"summary"},"Summary"),(0,i.kt)("p",null,"It is observed that the deployment of a project with more than 1000 jobs took around 6 minutes to complete, and the\nreplay request for the same project took around 4 minutes. An analysis to improve the time taken for this is needed,\nespecially if the project will be broke down to multiple namespaces. This will cause problem, as 10 namespaces might\ncan consume 6 minutes * 10 times."),(0,i.kt)("h1",{id:"technical-design"},"Technical Design"),(0,i.kt)("h2",{id:"background-"},"Background :"),(0,i.kt)("p",null,"Understanding the current process of the mentioned issues:"),(0,i.kt)("h3",{id:"deploy"},"Deploy"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolving dependencies for all the jobs in the requested project"),(0,i.kt)("li",{parentName:"ul"},"Resolving priorities for all the jobs"),(0,i.kt)("li",{parentName:"ul"},"Compiling the jobs within requested namespace"),(0,i.kt)("li",{parentName:"ul"},"Uploading compiled jobs to storage")),(0,i.kt)("h3",{id:"replay-request"},"Replay Request"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolving dependencies for all the jobs in the requested project"),(0,i.kt)("li",{parentName:"ul"},"Clearing scheduler dag run(s)")),(0,i.kt)("h3",{id:"backup-request"},"Backup Request"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolving dependencies for all the jobs in the requested project"),(0,i.kt)("li",{parentName:"ul"},"Duplicate table")),(0,i.kt)("p",null,"All the processes above need dependency resolution. When resolving dependency, it is being done for ALL the jobs in the\nproject, regardless of the namespace and regardless if it has changed or not. For every job (bq2bq for example), Optimus\nwill call each of the jobs the GenerateDependencies function in the plugin, and do a dry run hitting the Bigquery API.\nThis process has been done in parallel."),(0,i.kt)("p",null,"To simulate, let\u2019s say there are 1000 bq2bq jobs in a project."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"}," Job | Upstream\n--------------------------\n A | -\n B | A\n C | A, B\n D | C \n ... | ...\n")),(0,i.kt)("p",null,"There is a change in job C, that it no longer has dependency to job A. When it happens, when deploying, currently\nOptimus will resolve dependencies for all 1000 jobs. While in fact, the changed dependencies will only happen for job C.\nThere is only a slight difference where upstream is no longer A and B but only B."),(0,i.kt)("p",null,"Currently, Optimus is resolving dependencies every time it is needed because it is not stored anywhere, due to keep\nchanging dependencies. However, now we are seeing a timing problem, and the fact that not all jobs need to be dependency\nresolved everytime there\u2019s a deployment, a modification can be considered."),(0,i.kt)("p",null,"As part of this issue, we are also revisiting the current flow of job deployment process."),(0,i.kt)("h2",{id:"expected-behavior"},"Expected Behavior"),(0,i.kt)("h3",{id:"deploy-job"},"Deploy Job"),(0,i.kt)("p",null,"Accepts the whole state of the namespace/project. What is not available will be deleted."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Identify the added / modified / deleted jobs"),(0,i.kt)("li",{parentName:"ul"},"Resolve dependency only for the added or modified jobs and persist the dependency to DB"),(0,i.kt)("li",{parentName:"ul"},"Do priority resolution for all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Compile all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Upload the compiled jobs")),(0,i.kt)("p",null,"The difference between the expected behavior and current implementation is that we will only resolve dependency for\nwhat are necessary, and we will compile all the jobs in the project regardless the namespace. Compile and deploy all\njobs in the project is necessary to avoid below use case:"),(0,i.kt)("p",null,"Let's say in a single project, lies these 4 jobs. Job C depend on Job B, job B depend on Job A, and Job A and Job D are\nindependent. Notice that Job C is in a different namespace."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"Job A (Namespace 1) : weight 100\n|-- Job B (Namespace 1) : weight 90\n| |-- Job C (Namespace 2) : weight 80\n|-- Job D (Namespace 1) : weight 100\n")),(0,i.kt)("p",null,"Now let's say Job E (Namespace 1) is introduced and Job B is no longer depend directly on Job A, but instead to the new\nJob E."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"Job A (Namespace 1) : weight 100\n|-- Job E (Namespace 1) : weight 90\n| |-- Job B (Namespace 1) : weight 80\n| |-- Job C (Namespace 2) : weight 70\n|-- Job D (Namespace 1) : weight 100\n")),(0,i.kt)("p",null,"Notice that Job C priority weight has been changed. This example shows that even though the changes are in Namespace 1,\nthe other namespace is also affected and needs to be recompiled and deployed."),(0,i.kt)("h3",{id:"create-job"},"Create Job"),(0,i.kt)("p",null,"Accept a single/multiple jobs to be created and deployed."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolve dependency for the requested job and persist the dependency to DB"),(0,i.kt)("li",{parentName:"ul"},"Do priority resolution for all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Compile all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Upload the compiled jobs")),(0,i.kt)("h3",{id:"delete-job"},"Delete Job"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Identify any dependent jobs using dependencies table"),(0,i.kt)("li",{parentName:"ul"},"Delete only if there are no dependencies\nTBD: Doing soft delete or move the deleted jobs to a different table")),(0,i.kt)("h3",{id:"refresh-job"},"Refresh Job"),(0,i.kt)("p",null,"Using current state of job that has been stored, redo the dependency resolution, recompile, redeploy.\nCan be useful to do clean deploy or upgrading jobs."),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Resolve dependency for all jobs in the namespace/project and persist to DB"),(0,i.kt)("li",{parentName:"ul"},"Do priority resolution for all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Compile all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Upload the compiled jobs")),(0,i.kt)("h3",{id:"create-resource"},"Create Resource"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Deploy the requested resource"),(0,i.kt)("li",{parentName:"ul"},"Identify jobs that are dependent to the resource"),(0,i.kt)("li",{parentName:"ul"},"Resolve dependency for the jobs found"),(0,i.kt)("li",{parentName:"ul"},"Compile all jobs in the project"),(0,i.kt)("li",{parentName:"ul"},"Upload the compiled jobs\nAn explanation of this behaviour can be found in ",(0,i.kt)("inlineCode",{parentName:"li"},"Handling Modified View")," section")),(0,i.kt)("h3",{id:"delete-resource"},"Delete Resource"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Identify jobs that are dependent to the requested resource"),(0,i.kt)("li",{parentName:"ul"},"Delete only if there are no dependencies")),(0,i.kt)("h3",{id:"replay--backup"},"Replay & Backup"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Get the dependencies from the dependencies table."),(0,i.kt)("li",{parentName:"ul"},"Continue to build the tree.")),(0,i.kt)("h2",{id:"approach-"},"Approach :"),(0,i.kt)("h3",{id:"checking-which-jobs-are-modified"},"Checking which jobs are modified?"),(0,i.kt)("p",null,"Currently, Optimus receives all the jobs to be deployed, compares which one to be deleted and which one to keep,\nresolves and compiles them all. Optimus does not know the state of which changed."),(0,i.kt)("p",null,"One of the possibilities is by using Job hash. Fetch the jobs from DB, hash and compare with the one requested."),(0,i.kt)("h3",{id:"persistence"},"Persistence"),(0,i.kt)("p",null,"The process can be optimized only if the dependencies are stored, so no need to resolve it all every time it is needed.\nCurrently, this is the struct of JobSpec in Optimus:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-go"},"type JobSpec struct {\n ID uuid.UUID\n Name string\n Dependencies map[string]JobSpecDependency\n ....\n}\n\ntype JobSpecDependency struct {\n Project *ProjectSpec\n Job *JobSpec\n Type JobSpecDependencyType\n}\n")),(0,i.kt)("p",null,"The Dependencies field will be filled with inferred dependency after dependency resolution is finished."),(0,i.kt)("p",null,"We can have a new table to persist the job ID dependency."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"}," job_id | UUID\n job_dependency_id | UUID\n")),(0,i.kt)("p",null,"Example"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"}," Job | Upstream\n---------------------------\n A | -\n B | A\n C | A\n C | B\n D | C\n ... | ...\n")),(0,i.kt)("p",null,"If now C has been modified to have upstream of only B, means:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Record with jobID C will be deleted"),(0,i.kt)("li",{parentName:"ul"},"Insert 1 new record: C with dependency B")),(0,i.kt)("p",null,"Advantages:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Data is available even though there are pod restarts."),(0,i.kt)("li",{parentName:"ul"},"Better visibility of current dependencies.")),(0,i.kt)("p",null,"Disadvantages:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Additional time to write/read from DB")),(0,i.kt)("h3",{id:"event-based-mechanism-in-deployment"},"Event-Based Mechanism in Deployment"),(0,i.kt)("p",null,"Revisiting the process of deployment:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"Step | Deploy | Create Job | Refresh\n---------------------------------------------------------------\nResolve dependency | Diff | Requested | All\nResolve priority | All | All | All\nCompile | All | All | All\nUpload | All | All | All\n")),(0,i.kt)("p",null,"Notice that priority resolution, compilation, and upload compiled jobs needs to be done for all the jobs in the project\nfor all the namespaces. Each of the request can be done multiple times per minute and improvisation to speed up the\nprocess is needed."),(0,i.kt)("p",null,"Whenever there is a request to do deployment, job creation, and refresh, Optimus will do dependency resolution based\non each of the cases. After it finishes, it will push an event to be picked by a worker to do priority resolution,\ncompilation, and upload asynchronously. There will be deduplication in the event coming in, to avoid doing duplicated\nprocess."),(0,i.kt)("p",null,"There will be a get deployment status API introduced to poll whether these async processes has been finished or not."),(0,i.kt)("h3",{id:"handling-dependency-resolution-failure"},"Handling Dependency Resolution Failure"),(0,i.kt)("p",null,"Currently, whenever there is a single jobs that is failing in dependency resolution, and it is within the same\nnamespace as requested, it will fail the entire process. We are avoiding the entire deployment pipeline to be blocked\nby a single job failure, but instead sending it as part of the response and proceeding the deployment until finished.\nOnly the failed jobs will not be deployed. There will be metrics being added to add more visibility around this."),(0,i.kt)("h3",{id:"handling-modified-view"},"Handling Modified View"),(0,i.kt)("p",null,"A BQ2BQ job can have a source from views. For this job, the dependency will be the underlying tables of the view. Let's\nsimulate a case where there is a change in the view source."),(0,i.kt)("p",null,"In a project, there is view ",(0,i.kt)("inlineCode",{parentName:"p"},"X")," that querying from table ",(0,i.kt)("inlineCode",{parentName:"p"},"A")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"B"),". There is also table ",(0,i.kt)("inlineCode",{parentName:"p"},"C")," that querying from View\n",(0,i.kt)("inlineCode",{parentName:"p"},"X"),". The job dependencies for this case can be summarized as:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"}," Job | Upstream\n---------------------------\n A | -\n B | -\n C | A\n C | B\n")),(0,i.kt)("p",null,"Job C has dependency to job ",(0,i.kt)("inlineCode",{parentName:"p"},"A")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"B"),", even though it is querying from view ",(0,i.kt)("inlineCode",{parentName:"p"},"X"),"."),(0,i.kt)("p",null,"Imagine a case where view ",(0,i.kt)("inlineCode",{parentName:"p"},"X")," is modified, for example no longer querying from ",(0,i.kt)("inlineCode",{parentName:"p"},"A")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"B"),", but instead only from ",(0,i.kt)("inlineCode",{parentName:"p"},"A"),".\nJob ",(0,i.kt)("inlineCode",{parentName:"p"},"C")," dependency will never be updated, since it is not considered as modified. There should be a mechanism where if\na view is updated, it will also resolve the dependency for the jobs that depend on the view."),(0,i.kt)("p",null,"To make this happen, there should be a visibility of which resources are the sources of a job, for example which job is\nusing this view as a destination and querying from this view. Optimus is a transformation tool, in the job spec we store\nwhat is the transformation destination of the job. However, we are not storing what are the sources of the transformation.\nThe only thing we have is job dependency, not resource."),(0,i.kt)("p",null,"We can add a Source URNs field to the jobs specs, or create a Job Source table. Whenever there is a change in a view\nthrough Optimus, datastore should be able to request the dependency resolution for the view's dependent and having the\ndependencies updated. We will also provide the mechanism to refresh jobs."),(0,i.kt)("h3",{id:"cli-perspective"},"CLI Perspective"),(0,i.kt)("p",null,"Deploy job per namespace (using DeployJobSpecification rpc)"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"optimus job deploy --namespace --project\n")),(0,i.kt)("p",null,"Deploy job for selected jobs (using CreateJobSpecification rpc)"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"optimus job deploy --namespace --project --jobs=(job1,job2)\n")),(0,i.kt)("p",null,"Refresh the entire namespace/project"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"optimus job refresh --namespace --project\n")),(0,i.kt)("p",null,"Refresh the selected jobs"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"optimus job refresh --namespace --project --jobs=(job1,job2)\n")),(0,i.kt)("h2",{id:"other-thoughts"},"Other Thoughts:"),(0,i.kt)("p",null,"Cache Considerations instead of persisting to PG"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Might be faster as there is no additional time for write/read from DB"),(0,i.kt)("li",{parentName:"ul"},"Data will be unavailable post pod restarts. Need to redo the dependency resolution overall"),(0,i.kt)("li",{parentName:"ul"},"Poor visibility")))}u.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/7c550269.e31700b4.js b/assets/js/7c550269.3e410d26.js similarity index 99% rename from assets/js/7c550269.e31700b4.js rename to assets/js/7c550269.3e410d26.js index 3b2d42f6a6..82e6c5dfa2 100644 --- a/assets/js/7c550269.e31700b4.js +++ b/assets/js/7c550269.3e410d26.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[253],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>f});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),c=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=c(e.components);return a.createElement(s.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,f=u["".concat(s,".").concat(d)]||u[d]||m[d]||o;return n?a.createElement(f,i(i({ref:t},p),{},{components:n})):a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,i=new Array(o);i[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[u]="string"==typeof e?e:r,i[1]=l;for(var c=2;c{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>m,frontMatter:()=>o,metadata:()=>l,toc:()=>c});var a=n(7462),r=(n(7294),n(3905));const o={},i="Configuration",l={unversionedId:"client-guide/configuration",id:"client-guide/configuration",title:"Configuration",description:"Client configuration holds the necessary information for connecting to the Optimus server as well as for specification",source:"@site/docs/client-guide/configuration.md",sourceDirName:"client-guide",slug:"/client-guide/configuration",permalink:"/optimus/docs/client-guide/configuration",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/configuration.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"DB Migrations",permalink:"/optimus/docs/server-guide/db-migrations"},next:{title:"Managing Project & Namespace",permalink:"/optimus/docs/client-guide/managing-project-namespace"}},s={},c=[{value:"Project",id:"project",level:2},{value:"Preset (since v0.10.0)",id:"preset-since-v0100",level:3},{value:"Namespaces",id:"namespaces",level:2}],p={toc:c},u="wrapper";function m(e){let{components:t,...n}=e;return(0,r.kt)(u,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"configuration"},"Configuration"),(0,r.kt)("p",null,"Client configuration holds the necessary information for connecting to the Optimus server as well as for specification\ncreation. Optimus provides a way for you to initialize the client configuration by using the ",(0,r.kt)("inlineCode",{parentName:"p"},"init")," command. Go to the\ndirectory where you want to have your Optimus specifications. Run the below command and answer the prompt questions:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus init\n\n? What is the Optimus service host? localhost:9100\n? What is the Optimus project name? sample_project\n? What is the namespace name? sample_namespace\n? What is the type of data store for this namespace? bigquery\n? Do you want to add another namespace? No\nClient config is initialized successfully\n")),(0,r.kt)("p",null,"After running the init command, the Optimus client config will be configured. Along with it, the directories for the\nchosen namespaces, including the sub-directories for jobs and resources will be created with the following structure:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"},"sample_project\n\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n\u2502 \u2514\u2500\u2500 resources\n\u2514\u2500\u2500 optimus.yaml\n")),(0,r.kt)("p",null,"Below is the client configuration that has been generated:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nlog:\n level: INFO\n format: ""\nhost: localhost:9100\nproject:\n name: sample_project\n config: {}\nnamespaces:\n- name: sample_namespace\n config: {}\n job:\n path: sample_namespace/jobs\n datastore:\n - type: bigquery\n path: sample_namespace/resources\n backup: {}\n')),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Configuration"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Version"),(0,r.kt)("td",{parentName:"tr",align:null},"Supports only version 1 at the moment.")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Log"),(0,r.kt)("td",{parentName:"tr",align:null},"Logging level & format configuration")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Host"),(0,r.kt)("td",{parentName:"tr",align:null},"Optimus server host")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Project"),(0,r.kt)("td",{parentName:"tr",align:null},"Chosen Optimus project name and configurations.")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Namespaces"),(0,r.kt)("td",{parentName:"tr",align:null},"Namespaces that are owned by the project.")))),(0,r.kt)("h2",{id:"project"},"Project"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Project name should be unique."),(0,r.kt)("li",{parentName:"ul"},"Several configs are mandatory for job compilation and deployment use case:",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"storage_path")," config to store the job compilation result. A path can be anything, for example, a local directory\npath or a Google Cloud Storage path."),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"scheduler_host")," being used for job execution and sensors."),(0,r.kt)("li",{parentName:"ul"},"Specific secrets might be needed for the above configs. Take a look at the detail ",(0,r.kt)("a",{parentName:"li",href:"/optimus/docs/client-guide/managing-secrets"},"here"),"."))),(0,r.kt)("li",{parentName:"ul"},"Several configs are optional:",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"scheduler_version")," to define the scheduler version. More detail is explained ",(0,r.kt)("a",{parentName:"li",href:"/optimus/docs/client-guide/defining-scheduler-version"},"here"),"."))),(0,r.kt)("li",{parentName:"ul"},"You can put any other project configurations which can be used in job specifications.")),(0,r.kt)("h3",{id:"preset-since-v0100"},"Preset (since v0.10.0)"),(0,r.kt)("p",null,"Window preset can be configured within the specified project. Preset allows for easier usage of window configuration. For more information, please refer to ",(0,r.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/intervals-and-windows"},"this page"),"."),(0,r.kt)("h2",{id:"namespaces"},"Namespaces"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Name should be unique in the project."),(0,r.kt)("li",{parentName:"ul"},"You can put any namespace configurations which can be used in specifications."),(0,r.kt)("li",{parentName:"ul"},"Job path needs to be properly set so Optimus CLI will able to find all of your job specifications to be processed."),(0,r.kt)("li",{parentName:"ul"},"For datastore, currently Optimus only accepts ",(0,r.kt)("inlineCode",{parentName:"li"},"bigquery")," datastore type and you need to set the specification path\nfor this. Also, there is an optional ",(0,r.kt)("inlineCode",{parentName:"li"},"backup")," config map. Take a look at the backup guide section ",(0,r.kt)("a",{parentName:"li",href:"/optimus/docs/client-guide/backup-bigquery-resource"},"here"),"\nto understand more about this.")))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[253],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>f});var a=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),c=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=c(e.components);return a.createElement(s.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=c(n),d=r,f=u["".concat(s,".").concat(d)]||u[d]||m[d]||o;return n?a.createElement(f,i(i({ref:t},p),{},{components:n})):a.createElement(f,i({ref:t},p))}));function f(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,i=new Array(o);i[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[u]="string"==typeof e?e:r,i[1]=l;for(var c=2;c{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>m,frontMatter:()=>o,metadata:()=>l,toc:()=>c});var a=n(7462),r=(n(7294),n(3905));const o={},i="Configuration",l={unversionedId:"client-guide/configuration",id:"client-guide/configuration",title:"Configuration",description:"Client configuration holds the necessary information for connecting to the Optimus server as well as for specification",source:"@site/docs/client-guide/configuration.md",sourceDirName:"client-guide",slug:"/client-guide/configuration",permalink:"/optimus/docs/client-guide/configuration",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/configuration.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"DB Migrations",permalink:"/optimus/docs/server-guide/db-migrations"},next:{title:"Managing Project & Namespace",permalink:"/optimus/docs/client-guide/managing-project-namespace"}},s={},c=[{value:"Project",id:"project",level:2},{value:"Preset (since v0.10.0)",id:"preset-since-v0100",level:3},{value:"Namespaces",id:"namespaces",level:2}],p={toc:c},u="wrapper";function m(e){let{components:t,...n}=e;return(0,r.kt)(u,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"configuration"},"Configuration"),(0,r.kt)("p",null,"Client configuration holds the necessary information for connecting to the Optimus server as well as for specification\ncreation. Optimus provides a way for you to initialize the client configuration by using the ",(0,r.kt)("inlineCode",{parentName:"p"},"init")," command. Go to the\ndirectory where you want to have your Optimus specifications. Run the below command and answer the prompt questions:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus init\n\n? What is the Optimus service host? localhost:9100\n? What is the Optimus project name? sample_project\n? What is the namespace name? sample_namespace\n? What is the type of data store for this namespace? bigquery\n? Do you want to add another namespace? No\nClient config is initialized successfully\n")),(0,r.kt)("p",null,"After running the init command, the Optimus client config will be configured. Along with it, the directories for the\nchosen namespaces, including the sub-directories for jobs and resources will be created with the following structure:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"},"sample_project\n\u251c\u2500\u2500 sample_namespace\n\u2502 \u2514\u2500\u2500 jobs\n\u2502 \u2514\u2500\u2500 resources\n\u2514\u2500\u2500 optimus.yaml\n")),(0,r.kt)("p",null,"Below is the client configuration that has been generated:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nlog:\n level: INFO\n format: ""\nhost: localhost:9100\nproject:\n name: sample_project\n config: {}\nnamespaces:\n- name: sample_namespace\n config: {}\n job:\n path: sample_namespace/jobs\n datastore:\n - type: bigquery\n path: sample_namespace/resources\n backup: {}\n')),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Configuration"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Version"),(0,r.kt)("td",{parentName:"tr",align:null},"Supports only version 1 at the moment.")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Log"),(0,r.kt)("td",{parentName:"tr",align:null},"Logging level & format configuration")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Host"),(0,r.kt)("td",{parentName:"tr",align:null},"Optimus server host")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Project"),(0,r.kt)("td",{parentName:"tr",align:null},"Chosen Optimus project name and configurations.")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"Namespaces"),(0,r.kt)("td",{parentName:"tr",align:null},"Namespaces that are owned by the project.")))),(0,r.kt)("h2",{id:"project"},"Project"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Project name should be unique."),(0,r.kt)("li",{parentName:"ul"},"Several configs are mandatory for job compilation and deployment use case:",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"storage_path")," config to store the job compilation result. A path can be anything, for example, a local directory\npath or a Google Cloud Storage path."),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"scheduler_host")," being used for job execution and sensors."),(0,r.kt)("li",{parentName:"ul"},"Specific secrets might be needed for the above configs. Take a look at the detail ",(0,r.kt)("a",{parentName:"li",href:"/optimus/docs/client-guide/managing-secrets"},"here"),"."))),(0,r.kt)("li",{parentName:"ul"},"Several configs are optional:",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"scheduler_version")," to define the scheduler version. More detail is explained ",(0,r.kt)("a",{parentName:"li",href:"/optimus/docs/client-guide/defining-scheduler-version"},"here"),"."))),(0,r.kt)("li",{parentName:"ul"},"You can put any other project configurations which can be used in job specifications.")),(0,r.kt)("h3",{id:"preset-since-v0100"},"Preset (since v0.10.0)"),(0,r.kt)("p",null,"Window preset can be configured within the specified project. Preset allows for easier usage of window configuration. For more information, please refer to ",(0,r.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/intervals-and-windows"},"this page"),"."),(0,r.kt)("h2",{id:"namespaces"},"Namespaces"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Name should be unique in the project."),(0,r.kt)("li",{parentName:"ul"},"You can put any namespace configurations which can be used in specifications."),(0,r.kt)("li",{parentName:"ul"},"Job path needs to be properly set so Optimus CLI will able to find all of your job specifications to be processed."),(0,r.kt)("li",{parentName:"ul"},"For datastore, currently Optimus only accepts ",(0,r.kt)("inlineCode",{parentName:"li"},"bigquery")," datastore type and you need to set the specification path\nfor this. Also, there is an optional ",(0,r.kt)("inlineCode",{parentName:"li"},"backup")," config map. Take a look at the backup guide section ",(0,r.kt)("a",{parentName:"li",href:"/optimus/docs/client-guide/backup-bigquery-resource"},"here"),"\nto understand more about this.")))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/7fa9dab1.abec0c1c.js b/assets/js/7fa9dab1.11b74139.js similarity index 99% rename from assets/js/7fa9dab1.abec0c1c.js rename to assets/js/7fa9dab1.11b74139.js index c380bf8991..a24b90483c 100644 --- a/assets/js/7fa9dab1.abec0c1c.js +++ b/assets/js/7fa9dab1.11b74139.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9047],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>h});var a=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=a.createContext({}),d=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=d(e.components);return a.createElement(s.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=d(n),c=o,h=u["".concat(s,".").concat(c)]||u[c]||m[c]||r;return n?a.createElement(h,i(i({ref:t},p),{},{components:n})):a.createElement(h,i({ref:t},p))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=c;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[u]="string"==typeof e?e:o,i[1]=l;for(var d=2;d{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>m,frontMatter:()=>r,metadata:()=>l,toc:()=>d});var a=n(7462),o=(n(7294),n(3905));const r={},i="Intervals and Windows",l={unversionedId:"concepts/intervals-and-windows",id:"concepts/intervals-and-windows",title:"Intervals and Windows",description:"When defining a new job, you need to define the interval (cron) at which it will be triggered. This parameter can give",source:"@site/docs/concepts/intervals-and-windows.md",sourceDirName:"concepts",slug:"/concepts/intervals-and-windows",permalink:"/optimus/docs/concepts/intervals-and-windows",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/intervals-and-windows.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Macros",permalink:"/optimus/docs/concepts/macros"},next:{title:"Secret",permalink:"/optimus/docs/concepts/secret"}},s={},d=[{value:"Window Configuration",id:"window-configuration",level:2},{value:"Custom Window",id:"custom-window",level:3},{value:"Window Preset (since v0.10.0)",id:"window-preset-since-v0100",level:3}],p={toc:d},u="wrapper";function m(e){let{components:t,...n}=e;return(0,o.kt)(u,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"intervals-and-windows"},"Intervals and Windows"),(0,o.kt)("p",null,"When defining a new job, you need to define the ",(0,o.kt)("strong",{parentName:"p"},"interval (cron)")," at which it will be triggered. This parameter can give\nyou a precise value when the job is scheduled for execution but only a rough estimate exactly when the job is executing.\nIt is very common in a ETL pipeline to know when the job is exactly executing as well as for what time window the current\ntransformation will consume the data."),(0,o.kt)("p",null,"For example, assume there is a job that querying from a table using below statement:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},"SELECT * FROM table WHERE\ncreated_at >= DATE('{{.DSTART}}') AND\ncreated_at < DATE('{{.DEND}}')\n")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},(0,o.kt)("em",{parentName:"strong"},"DSTART"))," and ",(0,o.kt)("strong",{parentName:"p"},(0,o.kt)("em",{parentName:"strong"},"DEND"))," could be replaced at the time of compilation with based on its window configuration.\nWithout the provided filter, we will have to consume all the records which are created till date inside the table\neven though the previous rows might already been processed."),(0,o.kt)("p",null,"These ",(0,o.kt)("em",{parentName:"p"},"DSTART")," and ",(0,o.kt)("em",{parentName:"p"},"DEND")," values of the input window could vary depending on the ETL job requirement."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"For a simple transformation job executing daily, it would need to consume full day work of yesterday\u2019s data."),(0,o.kt)("li",{parentName:"ul"},"A job might be consuming data for a week/month for an aggregation job, but the data boundaries should be complete,\nnot consuming any partial data of a day.")),(0,o.kt)("h2",{id:"window-configuration"},"Window Configuration"),(0,o.kt)("p",null,"Optimus allows user to define the amount of data window to consume through window configurations. The configurations\nact on the schedule",(0,o.kt)("em",{parentName:"p"},"time of the job and applied in order to compute _DSTART")," and ",(0,o.kt)("em",{parentName:"p"},"DEND"),"."),(0,o.kt)("p",null,"The following is the list of available confiugration the user can setup a window:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Truncate_to"),': The data window on most of the scenarios needs to be aligned to a well-defined time window\nlike month start to month end, or week start to weekend with week start being monday, or a complete day.\nInorder to achieve that the truncate_to option is provided which can be configured with either of these values\n"h", "d", "w", "M" through which for a given schedule_time the end_time will be the end of last hour, day, week, month respectively.'),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Offset"),": Offset is time duration configuration which enables user to move the ",(0,o.kt)("inlineCode",{parentName:"li"},"end_time"),' post truncation.\nUser can define the duration like "24h", "2h45m", "60s", "-45m24h", "0", "", "2M", "45M24h", "45M24h30m"\nwhere "h","m","s","M" means hour, month, seconds, Month respectively.'),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Size"),": Size enables user to define the amount of data to consume from the ",(0,o.kt)("inlineCode",{parentName:"li"},"end_time")," again defined through the duration same as offset.")),(0,o.kt)("p",null,"To further understand, the following is an example with its explanation. ",(0,o.kt)("strong",{parentName:"p"},"Important")," note, the following example uses\nwindow ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 2")," because ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 1")," will soon be deprecated."),(0,o.kt)("p",null,"For example, previous-mentioned job has ",(0,o.kt)("inlineCode",{parentName:"p"},"0 2 * * *")," schedule interval and is scheduled to run on\n",(0,o.kt)("strong",{parentName:"p"},"2023-03-07 at 02.00 UTC")," with following details:"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Configuration"),(0,o.kt)("th",{parentName:"tr",align:null},"Value"),(0,o.kt)("th",{parentName:"tr",align:null},"Description"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Truncate_to"),(0,o.kt)("td",{parentName:"tr",align:null},"d"),(0,o.kt)("td",{parentName:"tr",align:null},"Even though it is scheduled at 02.00 AM, data window will be day-truncated (00.00 AM).")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Offset"),(0,o.kt)("td",{parentName:"tr",align:null},"-24h"),(0,o.kt)("td",{parentName:"tr",align:null},"Shifts the window to be 1 day earlier.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Size"),(0,o.kt)("td",{parentName:"tr",align:null},"24h"),(0,o.kt)("td",{parentName:"tr",align:null},"Gap between DSTART and DEND is 24h.")))),(0,o.kt)("p",null,"Above configuration will produce below window:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("em",{parentName:"li"},"DSTART"),": 2023-03-05T00:00:00Z"),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("em",{parentName:"li"},"DEND"),": 2023-03-06T00:00:00Z")),(0,o.kt)("p",null,"This means, the query will be compiled to the following query"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},"SELECT * FROM table WHERE\ncreated_at >= DATE('2023-03-05T00:00:00Z') AND\ncreated_at < DATE('2023-03-06T00:00:00Z')\n")),(0,o.kt)("p",null,"Assume the table content is as the following:"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"name"),(0,o.kt)("th",{parentName:"tr",align:null},"created_at"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Rick"),(0,o.kt)("td",{parentName:"tr",align:null},"2023-03-05")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Sanchez"),(0,o.kt)("td",{parentName:"tr",align:null},"2023-03-06")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Serious"),(0,o.kt)("td",{parentName:"tr",align:null},"2023-03-07")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Sam"),(0,o.kt)("td",{parentName:"tr",align:null},"2023-03-07")))),(0,o.kt)("p",null,"When the job that scheduled at ",(0,o.kt)("strong",{parentName:"p"},"2023-03-07")," runs, the job will consume ",(0,o.kt)("inlineCode",{parentName:"p"},"Rick")," as the input of the table."),(0,o.kt)("p",null,"Window configuration can be specified in two ways, through custom window configuration and through window preset."),(0,o.kt)("h3",{id:"custom-window"},"Custom Window"),(0,o.kt)("p",null,"Through this option, the user can directly configure the window that meets their requirement in the job spec YAML.\nThe following is an example of its usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1 # decides window version\nname: sample-project.playground.table1\nowner: sample_owner\nschedule:\n ...\nbehavior:\n ...\ntask:\n name: bq2bq\n config:\n ...\n window:\n size: 24h\n offset: "0"\n truncate_to: d\nlabels:\n ...\nhooks: []\ndependencies: []\n')),(0,o.kt)("p",null,"Notice the window configuration is specified under field ",(0,o.kt)("inlineCode",{parentName:"p"},"task.window"),". ",(0,o.kt)("strong",{parentName:"p"},"Important")," note, the ",(0,o.kt)("inlineCode",{parentName:"p"},"version")," field decides which\nversion of window capability to be used. Currently available is window ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 1")," and window ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 2"),". Version 2 is recommended\nto be used as verion 1 will soon be deprecated. To know the difference between the two version, run the ",(0,o.kt)("inlineCode",{parentName:"p"},"playground")," feature for window:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"optimus playground window\n")),(0,o.kt)("h3",{id:"window-preset-since-v0100"},"Window Preset (since v0.10.0)"),(0,o.kt)("p",null,"Window preset is a feature that allows easier setup of window configuration while also maintaining consistency. Through this feature,\nthe user can configure a definition of window once, then use it multiple times through the jobs which require it. ",(0,o.kt)("strong",{parentName:"p"},"Important")," note,\nwindow preset always use window ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 2"),". The main components of window preset are as follow."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Window Preset File"))),(0,o.kt)("p",null,"Presets configuration is put in a dedicated YAML file. The way to configure it still uses the same window configuration\nlike ",(0,o.kt)("inlineCode",{parentName:"p"},"truncate_to"),", ",(0,o.kt)("inlineCode",{parentName:"p"},"offset"),", and ",(0,o.kt)("inlineCode",{parentName:"p"},"size"),". Though, there are some additions, like the name of the preset and the description to explain this preset.\nThe following is an example of how to define a preset under ",(0,o.kt)("inlineCode",{parentName:"p"},"presets.yaml")," file (note that the file name does not have to be this one)."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"presets:\n yesterday:\n description: defines yesterday window\n window:\n truncate_to: d\n offset: -24h\n size: 24h\n last_month:\n description: defines last 30 days window\n window:\n truncate_to: M\n offset: -30d\n size: 30d\n")),(0,o.kt)("p",null,"In the above example, the file ",(0,o.kt)("inlineCode",{parentName:"p"},"presets.yaml")," defines two presets, named ",(0,o.kt)("inlineCode",{parentName:"p"},"yesterday")," and ",(0,o.kt)("inlineCode",{parentName:"p"},"last_month"),". The name of preset ",(0,o.kt)("strong",{parentName:"p"},"SHOULD")," be\nin lower case. All of the fields are required, unless specified otherwise."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Preset Reference under Project"))),(0,o.kt)("p",null,"If the preset file is already specified, the next thing to do is to ensure that the preset file is referenced under project configuration.\nThe following is an example to refer the preset file under project configuration:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"version: 1\nlog:\n ...\nhost: localhost:9100\nproject:\n name: development_project\n preset_path: ./preset.yaml # points to preset file\n config:\n ...\nnamespaces:\n ...\n")),(0,o.kt)("p",null,"In the above example, a new field is present, named ",(0,o.kt)("inlineCode",{parentName:"p"},"preset_path"),". This path refers to where the preset file is located."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Preset Reference for Job Specification"))),(0,o.kt)("p",null,"Now, if the other two components are met, where the window preset file is specified and this file is referenced by the project, it means\nit is ready to be used. And the way to use it is by referencing which preset to be used in whichever job requires it. The following is an example\nof its usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"version: 1 # preset always use window version 2\nname: sample-project.playground.table1\nowner: sample_owner\nschedule:\n ...\nbehavior:\n ...\ntask:\n name: bq2bq\n config:\n ...\n window:\n preset: yesterday\nlabels:\n ...\nhooks: []\ndependencies: []\n")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Important")," note, preset is optional in nature. It means that even if the preset is specified, the user can still use\nthe custom window configuration depending on their need."))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9047],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>h});var a=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=a.createContext({}),d=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=d(e.components);return a.createElement(s.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=d(n),c=o,h=u["".concat(s,".").concat(c)]||u[c]||m[c]||r;return n?a.createElement(h,i(i({ref:t},p),{},{components:n})):a.createElement(h,i({ref:t},p))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=c;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[u]="string"==typeof e?e:o,i[1]=l;for(var d=2;d{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>i,default:()=>m,frontMatter:()=>r,metadata:()=>l,toc:()=>d});var a=n(7462),o=(n(7294),n(3905));const r={},i="Intervals and Windows",l={unversionedId:"concepts/intervals-and-windows",id:"concepts/intervals-and-windows",title:"Intervals and Windows",description:"When defining a new job, you need to define the interval (cron) at which it will be triggered. This parameter can give",source:"@site/docs/concepts/intervals-and-windows.md",sourceDirName:"concepts",slug:"/concepts/intervals-and-windows",permalink:"/optimus/docs/concepts/intervals-and-windows",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/intervals-and-windows.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Macros",permalink:"/optimus/docs/concepts/macros"},next:{title:"Secret",permalink:"/optimus/docs/concepts/secret"}},s={},d=[{value:"Window Configuration",id:"window-configuration",level:2},{value:"Custom Window",id:"custom-window",level:3},{value:"Window Preset (since v0.10.0)",id:"window-preset-since-v0100",level:3}],p={toc:d},u="wrapper";function m(e){let{components:t,...n}=e;return(0,o.kt)(u,(0,a.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"intervals-and-windows"},"Intervals and Windows"),(0,o.kt)("p",null,"When defining a new job, you need to define the ",(0,o.kt)("strong",{parentName:"p"},"interval (cron)")," at which it will be triggered. This parameter can give\nyou a precise value when the job is scheduled for execution but only a rough estimate exactly when the job is executing.\nIt is very common in a ETL pipeline to know when the job is exactly executing as well as for what time window the current\ntransformation will consume the data."),(0,o.kt)("p",null,"For example, assume there is a job that querying from a table using below statement:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},"SELECT * FROM table WHERE\ncreated_at >= DATE('{{.DSTART}}') AND\ncreated_at < DATE('{{.DEND}}')\n")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},(0,o.kt)("em",{parentName:"strong"},"DSTART"))," and ",(0,o.kt)("strong",{parentName:"p"},(0,o.kt)("em",{parentName:"strong"},"DEND"))," could be replaced at the time of compilation with based on its window configuration.\nWithout the provided filter, we will have to consume all the records which are created till date inside the table\neven though the previous rows might already been processed."),(0,o.kt)("p",null,"These ",(0,o.kt)("em",{parentName:"p"},"DSTART")," and ",(0,o.kt)("em",{parentName:"p"},"DEND")," values of the input window could vary depending on the ETL job requirement."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"For a simple transformation job executing daily, it would need to consume full day work of yesterday\u2019s data."),(0,o.kt)("li",{parentName:"ul"},"A job might be consuming data for a week/month for an aggregation job, but the data boundaries should be complete,\nnot consuming any partial data of a day.")),(0,o.kt)("h2",{id:"window-configuration"},"Window Configuration"),(0,o.kt)("p",null,"Optimus allows user to define the amount of data window to consume through window configurations. The configurations\nact on the schedule",(0,o.kt)("em",{parentName:"p"},"time of the job and applied in order to compute _DSTART")," and ",(0,o.kt)("em",{parentName:"p"},"DEND"),"."),(0,o.kt)("p",null,"The following is the list of available confiugration the user can setup a window:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Truncate_to"),': The data window on most of the scenarios needs to be aligned to a well-defined time window\nlike month start to month end, or week start to weekend with week start being monday, or a complete day.\nInorder to achieve that the truncate_to option is provided which can be configured with either of these values\n"h", "d", "w", "M" through which for a given schedule_time the end_time will be the end of last hour, day, week, month respectively.'),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Offset"),": Offset is time duration configuration which enables user to move the ",(0,o.kt)("inlineCode",{parentName:"li"},"end_time"),' post truncation.\nUser can define the duration like "24h", "2h45m", "60s", "-45m24h", "0", "", "2M", "45M24h", "45M24h30m"\nwhere "h","m","s","M" means hour, month, seconds, Month respectively.'),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Size"),": Size enables user to define the amount of data to consume from the ",(0,o.kt)("inlineCode",{parentName:"li"},"end_time")," again defined through the duration same as offset.")),(0,o.kt)("p",null,"To further understand, the following is an example with its explanation. ",(0,o.kt)("strong",{parentName:"p"},"Important")," note, the following example uses\nwindow ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 2")," because ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 1")," will soon be deprecated."),(0,o.kt)("p",null,"For example, previous-mentioned job has ",(0,o.kt)("inlineCode",{parentName:"p"},"0 2 * * *")," schedule interval and is scheduled to run on\n",(0,o.kt)("strong",{parentName:"p"},"2023-03-07 at 02.00 UTC")," with following details:"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"Configuration"),(0,o.kt)("th",{parentName:"tr",align:null},"Value"),(0,o.kt)("th",{parentName:"tr",align:null},"Description"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Truncate_to"),(0,o.kt)("td",{parentName:"tr",align:null},"d"),(0,o.kt)("td",{parentName:"tr",align:null},"Even though it is scheduled at 02.00 AM, data window will be day-truncated (00.00 AM).")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Offset"),(0,o.kt)("td",{parentName:"tr",align:null},"-24h"),(0,o.kt)("td",{parentName:"tr",align:null},"Shifts the window to be 1 day earlier.")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Size"),(0,o.kt)("td",{parentName:"tr",align:null},"24h"),(0,o.kt)("td",{parentName:"tr",align:null},"Gap between DSTART and DEND is 24h.")))),(0,o.kt)("p",null,"Above configuration will produce below window:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("em",{parentName:"li"},"DSTART"),": 2023-03-05T00:00:00Z"),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("em",{parentName:"li"},"DEND"),": 2023-03-06T00:00:00Z")),(0,o.kt)("p",null,"This means, the query will be compiled to the following query"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-sql"},"SELECT * FROM table WHERE\ncreated_at >= DATE('2023-03-05T00:00:00Z') AND\ncreated_at < DATE('2023-03-06T00:00:00Z')\n")),(0,o.kt)("p",null,"Assume the table content is as the following:"),(0,o.kt)("table",null,(0,o.kt)("thead",{parentName:"table"},(0,o.kt)("tr",{parentName:"thead"},(0,o.kt)("th",{parentName:"tr",align:null},"name"),(0,o.kt)("th",{parentName:"tr",align:null},"created_at"))),(0,o.kt)("tbody",{parentName:"table"},(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Rick"),(0,o.kt)("td",{parentName:"tr",align:null},"2023-03-05")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Sanchez"),(0,o.kt)("td",{parentName:"tr",align:null},"2023-03-06")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Serious"),(0,o.kt)("td",{parentName:"tr",align:null},"2023-03-07")),(0,o.kt)("tr",{parentName:"tbody"},(0,o.kt)("td",{parentName:"tr",align:null},"Sam"),(0,o.kt)("td",{parentName:"tr",align:null},"2023-03-07")))),(0,o.kt)("p",null,"When the job that scheduled at ",(0,o.kt)("strong",{parentName:"p"},"2023-03-07")," runs, the job will consume ",(0,o.kt)("inlineCode",{parentName:"p"},"Rick")," as the input of the table."),(0,o.kt)("p",null,"Window configuration can be specified in two ways, through custom window configuration and through window preset."),(0,o.kt)("h3",{id:"custom-window"},"Custom Window"),(0,o.kt)("p",null,"Through this option, the user can directly configure the window that meets their requirement in the job spec YAML.\nThe following is an example of its usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1 # decides window version\nname: sample-project.playground.table1\nowner: sample_owner\nschedule:\n ...\nbehavior:\n ...\ntask:\n name: bq2bq\n config:\n ...\n window:\n size: 24h\n offset: "0"\n truncate_to: d\nlabels:\n ...\nhooks: []\ndependencies: []\n')),(0,o.kt)("p",null,"Notice the window configuration is specified under field ",(0,o.kt)("inlineCode",{parentName:"p"},"task.window"),". ",(0,o.kt)("strong",{parentName:"p"},"Important")," note, the ",(0,o.kt)("inlineCode",{parentName:"p"},"version")," field decides which\nversion of window capability to be used. Currently available is window ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 1")," and window ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 2"),". Version 2 is recommended\nto be used as verion 1 will soon be deprecated. To know the difference between the two version, run the ",(0,o.kt)("inlineCode",{parentName:"p"},"playground")," feature for window:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-bash"},"optimus playground window\n")),(0,o.kt)("h3",{id:"window-preset-since-v0100"},"Window Preset (since v0.10.0)"),(0,o.kt)("p",null,"Window preset is a feature that allows easier setup of window configuration while also maintaining consistency. Through this feature,\nthe user can configure a definition of window once, then use it multiple times through the jobs which require it. ",(0,o.kt)("strong",{parentName:"p"},"Important")," note,\nwindow preset always use window ",(0,o.kt)("inlineCode",{parentName:"p"},"version: 2"),". The main components of window preset are as follow."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Window Preset File"))),(0,o.kt)("p",null,"Presets configuration is put in a dedicated YAML file. The way to configure it still uses the same window configuration\nlike ",(0,o.kt)("inlineCode",{parentName:"p"},"truncate_to"),", ",(0,o.kt)("inlineCode",{parentName:"p"},"offset"),", and ",(0,o.kt)("inlineCode",{parentName:"p"},"size"),". Though, there are some additions, like the name of the preset and the description to explain this preset.\nThe following is an example of how to define a preset under ",(0,o.kt)("inlineCode",{parentName:"p"},"presets.yaml")," file (note that the file name does not have to be this one)."),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"presets:\n yesterday:\n description: defines yesterday window\n window:\n truncate_to: d\n offset: -24h\n size: 24h\n last_month:\n description: defines last 30 days window\n window:\n truncate_to: M\n offset: -30d\n size: 30d\n")),(0,o.kt)("p",null,"In the above example, the file ",(0,o.kt)("inlineCode",{parentName:"p"},"presets.yaml")," defines two presets, named ",(0,o.kt)("inlineCode",{parentName:"p"},"yesterday")," and ",(0,o.kt)("inlineCode",{parentName:"p"},"last_month"),". The name of preset ",(0,o.kt)("strong",{parentName:"p"},"SHOULD")," be\nin lower case. All of the fields are required, unless specified otherwise."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Preset Reference under Project"))),(0,o.kt)("p",null,"If the preset file is already specified, the next thing to do is to ensure that the preset file is referenced under project configuration.\nThe following is an example to refer the preset file under project configuration:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"version: 1\nlog:\n ...\nhost: localhost:9100\nproject:\n name: development_project\n preset_path: ./preset.yaml # points to preset file\n config:\n ...\nnamespaces:\n ...\n")),(0,o.kt)("p",null,"In the above example, a new field is present, named ",(0,o.kt)("inlineCode",{parentName:"p"},"preset_path"),". This path refers to where the preset file is located."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("strong",{parentName:"li"},"Preset Reference for Job Specification"))),(0,o.kt)("p",null,"Now, if the other two components are met, where the window preset file is specified and this file is referenced by the project, it means\nit is ready to be used. And the way to use it is by referencing which preset to be used in whichever job requires it. The following is an example\nof its usage:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"version: 1 # preset always use window version 2\nname: sample-project.playground.table1\nowner: sample_owner\nschedule:\n ...\nbehavior:\n ...\ntask:\n name: bq2bq\n config:\n ...\n window:\n preset: yesterday\nlabels:\n ...\nhooks: []\ndependencies: []\n")),(0,o.kt)("p",null,(0,o.kt)("strong",{parentName:"p"},"Important")," note, preset is optional in nature. It means that even if the preset is specified, the user can still use\nthe custom window configuration depending on their need."))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/81694fd1.0202bb1c.js b/assets/js/81694fd1.0fa2d851.js similarity index 99% rename from assets/js/81694fd1.0202bb1c.js rename to assets/js/81694fd1.0fa2d851.js index 8e1009e4a8..9174487ec7 100644 --- a/assets/js/81694fd1.0202bb1c.js +++ b/assets/js/81694fd1.0fa2d851.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6070],{3905:(e,n,t)=>{t.d(n,{Zo:()=>p,kt:()=>c});var i=t(7294);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function r(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);n&&(i=i.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,i)}return t}function l(e){for(var n=1;n=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var s=i.createContext({}),u=function(e){var n=i.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):l(l({},n),e)),t},p=function(e){var n=u(e.components);return i.createElement(s.Provider,{value:n},e.children)},m="mdxType",d={inlineCode:"code",wrapper:function(e){var n=e.children;return i.createElement(i.Fragment,{},n)}},g=i.forwardRef((function(e,n){var t=e.components,a=e.mdxType,r=e.originalType,s=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),m=u(t),g=a,c=m["".concat(s,".").concat(g)]||m[g]||d[g]||r;return t?i.createElement(c,l(l({ref:n},p),{},{components:t})):i.createElement(c,l({ref:n},p))}));function c(e,n){var t=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var r=t.length,l=new Array(r);l[0]=g;var o={};for(var s in n)hasOwnProperty.call(n,s)&&(o[s]=n[s]);o.originalType=e,o[m]="string"==typeof e?e:a,l[1]=o;for(var u=2;u{t.r(n),t.d(n,{assets:()=>s,contentTitle:()=>l,default:()=>d,frontMatter:()=>r,metadata:()=>o,toc:()=>u});var i=t(7462),a=(t(7294),t(3905));const r={},l="Introduction of Plugin Development",o={unversionedId:"building-plugin/introduction",id:"building-plugin/introduction",title:"Introduction of Plugin Development",description:"As mentioned in the concepts, plugins provide support for various data warehouses & any",source:"@site/docs/building-plugin/introduction.md",sourceDirName:"building-plugin",slug:"/building-plugin/introduction",permalink:"/optimus/docs/building-plugin/introduction",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/building-plugin/introduction.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Defining Scheduler Version",permalink:"/optimus/docs/client-guide/defining-scheduler-version"},next:{title:"Tutorial of Plugin Development",permalink:"/optimus/docs/building-plugin/tutorial"}},s={},u=[{value:"Yaml Implementation of Plugin",id:"yaml-implementation-of-plugin",level:2},{value:"Limitations of Yaml plugins:",id:"limitations-of-yaml-plugins",level:3},{value:"Validating Yaml plugins:",id:"validating-yaml-plugins",level:3},{value:"Binary Implementation of Plugin (to be deprecated)",id:"binary-implementation-of-plugin-to-be-deprecated",level:2}],p={toc:u},m="wrapper";function d(e){let{components:n,...t}=e;return(0,a.kt)(m,(0,i.Z)({},p,t,{components:n,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"introduction-of-plugin-development"},"Introduction of Plugin Development"),(0,a.kt)("p",null,"As mentioned in the ",(0,a.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/plugin"},"concepts"),", plugins provide support for various data warehouses & any\nthird party system to handle all the data transformations or movement. Before we start, let\u2019s take a look at different\nimplementations of plugin: YAML and binary."),(0,a.kt)("h2",{id:"yaml-implementation-of-plugin"},"Yaml Implementation of Plugin"),(0,a.kt)("p",null,"Most plugins are expected to implement just the info and project side use-cases (mentioned above) and these are\ndata-driven i.e., plugin just provide data to Optimus. To simplify the development process of plugins, support for\nyaml mode of defining plugins is added."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-go"},'// representation of a yaml plugin schema in golang\n\n// below struct definition in golang can be marshalled\n// to generate yaml plugins\n\ntype YamlPlugin struct {\n// info use-case\nName string `yaml:"name"`\nDescription string `yaml:"description"`\nPlugintype string `yaml:"plugintype"`\nPluginversion string `yaml:"pluginversion"`\nImage string `yaml:"image"`\n\n // survey use-case\n Questions []struct {\n Name string `yaml:"name"`\n Prompt string `yaml:"prompt"`\n Help string `yaml:"help"`\n Regexp string `yaml:"regexp"`\n Validationerror string `yaml:"validationerror"`\n Minlength int `yaml:"minlength"`\n Required bool `yaml:"required,omitempty"`\n Maxlength int `yaml:"maxlength,omitempty"`\n Subquestions []struct {\n Ifvalue string `yaml:"ifvalue"`\n Questions []struct {\n Name string `yaml:"name"`\n Prompt string `yaml:"prompt"`\n Help string `yaml:"help"`\n Multiselect []string `yaml:"multiselect"`\n Regexp string `yaml:"regexp"`\n Validationerror string `yaml:"validationerror"`\n Minlength int `yaml:"minlength"`\n Required bool `yaml:"required,omitempty"`\n Maxlength int `yaml:"maxlength,omitempty"`\n } `yaml:"questions"`\n } `yaml:"subquestions,omitempty"`\n } `yaml:"questions"`\n\n // default-static-values use-case\n Defaultassets []struct {\n Name string `yaml:"name"`\n Value string `yaml:"value"`\n } `yaml:"defaultassets"`\n Defaultconfig []struct {\n Name string `yaml:"name"`\n Value string `yaml:"value"`\n } `yaml:"defaultconfig"`\n}\n')),(0,a.kt)("p",null,"Refer to sample implementation here."),(0,a.kt)("h3",{id:"limitations-of-yaml-plugins"},"Limitations of Yaml plugins:"),(0,a.kt)("p",null,"Here the scope of YAML plugins is limited to driving surveys, providing default values for job config and assets, and\nproviding plugin info. As the majority of the plugins are expected to implement a subset of these use cases, the\nsupport for YAML definitions for plugins is added which simplifies the development, packaging, and distribution of plugins."),(0,a.kt)("p",null,"For plugins that require enriching Optimus server-side behavior, YAML definitions fall short as this would require some code."),(0,a.kt)("h3",{id:"validating-yaml-plugins"},"Validating Yaml plugins:"),(0,a.kt)("p",null,"Also support for validating yaml plugin is added into optimus. After creating yaml definitions of plugin, one can\nvalidate them as below:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"optimus plugin validate --path {{directory of yaml plugins}}\n")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"}," Note: The yaml plugin is expected to have file name as optimus-plugin-{{name}}.yaml\n")," Note: If Both yaml and binary plugin with same name are installed, Yaml implementation is prioritized over the\ncorresponding counterparts in binary implementation."),(0,a.kt)("h2",{id:"binary-implementation-of-plugin-to-be-deprecated"},"Binary Implementation of Plugin (to be deprecated)"),(0,a.kt)("p",null,"Binary implementations of Plugins are binaries which implement predefined protobuf interfaces to extend Optimus\nfunctionalities and augment the yaml implementations with executable code. Binary Plugins are implemented using\ngo-plugin developed by Hashicorp used in terraform and other similar products. Currently, Dependency Resolution Mod\nis the only interface that is supported in the binary approach to plugins."),(0,a.kt)("p",null,"Note : Binary plugins augment yaml plugins and they are not standalone."),(0,a.kt)("p",null,"Plugins can be implemented in any language as long as they can be exported as a single self-contained executable binary\nand implements a GRPC server. It is recommended to use Go currently for writing plugins because of its cross platform\nbuild functionality and to reuse protobuf sdk provided within Optimus core."),(0,a.kt)("p",null,(0,a.kt)("em",{parentName:"p"},"Binary Plugins can potentially modify the behavior of Optimus in undesired ways. Exercise caution when adding new\nplugins developed by unrecognized developers.")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6070],{3905:(e,n,t)=>{t.d(n,{Zo:()=>p,kt:()=>c});var i=t(7294);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function r(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);n&&(i=i.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,i)}return t}function l(e){for(var n=1;n=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var s=i.createContext({}),u=function(e){var n=i.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):l(l({},n),e)),t},p=function(e){var n=u(e.components);return i.createElement(s.Provider,{value:n},e.children)},m="mdxType",d={inlineCode:"code",wrapper:function(e){var n=e.children;return i.createElement(i.Fragment,{},n)}},g=i.forwardRef((function(e,n){var t=e.components,a=e.mdxType,r=e.originalType,s=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),m=u(t),g=a,c=m["".concat(s,".").concat(g)]||m[g]||d[g]||r;return t?i.createElement(c,l(l({ref:n},p),{},{components:t})):i.createElement(c,l({ref:n},p))}));function c(e,n){var t=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var r=t.length,l=new Array(r);l[0]=g;var o={};for(var s in n)hasOwnProperty.call(n,s)&&(o[s]=n[s]);o.originalType=e,o[m]="string"==typeof e?e:a,l[1]=o;for(var u=2;u{t.r(n),t.d(n,{assets:()=>s,contentTitle:()=>l,default:()=>d,frontMatter:()=>r,metadata:()=>o,toc:()=>u});var i=t(7462),a=(t(7294),t(3905));const r={},l="Introduction of Plugin Development",o={unversionedId:"building-plugin/introduction",id:"building-plugin/introduction",title:"Introduction of Plugin Development",description:"As mentioned in the concepts, plugins provide support for various data warehouses & any",source:"@site/docs/building-plugin/introduction.md",sourceDirName:"building-plugin",slug:"/building-plugin/introduction",permalink:"/optimus/docs/building-plugin/introduction",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/building-plugin/introduction.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Defining Scheduler Version",permalink:"/optimus/docs/client-guide/defining-scheduler-version"},next:{title:"Tutorial of Plugin Development",permalink:"/optimus/docs/building-plugin/tutorial"}},s={},u=[{value:"Yaml Implementation of Plugin",id:"yaml-implementation-of-plugin",level:2},{value:"Limitations of Yaml plugins:",id:"limitations-of-yaml-plugins",level:3},{value:"Validating Yaml plugins:",id:"validating-yaml-plugins",level:3},{value:"Binary Implementation of Plugin (to be deprecated)",id:"binary-implementation-of-plugin-to-be-deprecated",level:2}],p={toc:u},m="wrapper";function d(e){let{components:n,...t}=e;return(0,a.kt)(m,(0,i.Z)({},p,t,{components:n,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"introduction-of-plugin-development"},"Introduction of Plugin Development"),(0,a.kt)("p",null,"As mentioned in the ",(0,a.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/plugin"},"concepts"),", plugins provide support for various data warehouses & any\nthird party system to handle all the data transformations or movement. Before we start, let\u2019s take a look at different\nimplementations of plugin: YAML and binary."),(0,a.kt)("h2",{id:"yaml-implementation-of-plugin"},"Yaml Implementation of Plugin"),(0,a.kt)("p",null,"Most plugins are expected to implement just the info and project side use-cases (mentioned above) and these are\ndata-driven i.e., plugin just provide data to Optimus. To simplify the development process of plugins, support for\nyaml mode of defining plugins is added."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-go"},'// representation of a yaml plugin schema in golang\n\n// below struct definition in golang can be marshalled\n// to generate yaml plugins\n\ntype YamlPlugin struct {\n// info use-case\nName string `yaml:"name"`\nDescription string `yaml:"description"`\nPlugintype string `yaml:"plugintype"`\nPluginversion string `yaml:"pluginversion"`\nImage string `yaml:"image"`\n\n // survey use-case\n Questions []struct {\n Name string `yaml:"name"`\n Prompt string `yaml:"prompt"`\n Help string `yaml:"help"`\n Regexp string `yaml:"regexp"`\n Validationerror string `yaml:"validationerror"`\n Minlength int `yaml:"minlength"`\n Required bool `yaml:"required,omitempty"`\n Maxlength int `yaml:"maxlength,omitempty"`\n Subquestions []struct {\n Ifvalue string `yaml:"ifvalue"`\n Questions []struct {\n Name string `yaml:"name"`\n Prompt string `yaml:"prompt"`\n Help string `yaml:"help"`\n Multiselect []string `yaml:"multiselect"`\n Regexp string `yaml:"regexp"`\n Validationerror string `yaml:"validationerror"`\n Minlength int `yaml:"minlength"`\n Required bool `yaml:"required,omitempty"`\n Maxlength int `yaml:"maxlength,omitempty"`\n } `yaml:"questions"`\n } `yaml:"subquestions,omitempty"`\n } `yaml:"questions"`\n\n // default-static-values use-case\n Defaultassets []struct {\n Name string `yaml:"name"`\n Value string `yaml:"value"`\n } `yaml:"defaultassets"`\n Defaultconfig []struct {\n Name string `yaml:"name"`\n Value string `yaml:"value"`\n } `yaml:"defaultconfig"`\n}\n')),(0,a.kt)("p",null,"Refer to sample implementation here."),(0,a.kt)("h3",{id:"limitations-of-yaml-plugins"},"Limitations of Yaml plugins:"),(0,a.kt)("p",null,"Here the scope of YAML plugins is limited to driving surveys, providing default values for job config and assets, and\nproviding plugin info. As the majority of the plugins are expected to implement a subset of these use cases, the\nsupport for YAML definitions for plugins is added which simplifies the development, packaging, and distribution of plugins."),(0,a.kt)("p",null,"For plugins that require enriching Optimus server-side behavior, YAML definitions fall short as this would require some code."),(0,a.kt)("h3",{id:"validating-yaml-plugins"},"Validating Yaml plugins:"),(0,a.kt)("p",null,"Also support for validating yaml plugin is added into optimus. After creating yaml definitions of plugin, one can\nvalidate them as below:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"optimus plugin validate --path {{directory of yaml plugins}}\n")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"}," Note: The yaml plugin is expected to have file name as optimus-plugin-{{name}}.yaml\n")," Note: If Both yaml and binary plugin with same name are installed, Yaml implementation is prioritized over the\ncorresponding counterparts in binary implementation."),(0,a.kt)("h2",{id:"binary-implementation-of-plugin-to-be-deprecated"},"Binary Implementation of Plugin (to be deprecated)"),(0,a.kt)("p",null,"Binary implementations of Plugins are binaries which implement predefined protobuf interfaces to extend Optimus\nfunctionalities and augment the yaml implementations with executable code. Binary Plugins are implemented using\ngo-plugin developed by Hashicorp used in terraform and other similar products. Currently, Dependency Resolution Mod\nis the only interface that is supported in the binary approach to plugins."),(0,a.kt)("p",null,"Note : Binary plugins augment yaml plugins and they are not standalone."),(0,a.kt)("p",null,"Plugins can be implemented in any language as long as they can be exported as a single self-contained executable binary\nand implements a GRPC server. It is recommended to use Go currently for writing plugins because of its cross platform\nbuild functionality and to reuse protobuf sdk provided within Optimus core."),(0,a.kt)("p",null,(0,a.kt)("em",{parentName:"p"},"Binary Plugins can potentially modify the behavior of Optimus in undesired ways. Exercise caution when adding new\nplugins developed by unrecognized developers.")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/85be924b.c47e1e8b.js b/assets/js/85be924b.3bd1f4f0.js similarity index 98% rename from assets/js/85be924b.c47e1e8b.js rename to assets/js/85be924b.3bd1f4f0.js index adda69ac61..7ff2974914 100644 --- a/assets/js/85be924b.c47e1e8b.js +++ b/assets/js/85be924b.3bd1f4f0.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9861],{3905:(e,t,n)=>{n.d(t,{Zo:()=>l,kt:()=>f});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=r.createContext({}),s=function(e){var t=r.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},l=function(e){var t=s(e.components);return r.createElement(c.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,c=e.parentName,l=p(e,["components","mdxType","originalType","parentName"]),u=s(n),m=o,f=u["".concat(c,".").concat(m)]||u[m]||d[m]||a;return n?r.createElement(f,i(i({ref:t},l),{},{components:n})):r.createElement(f,i({ref:t},l))}));function f(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=m;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p[u]="string"==typeof e?e:o,i[1]=p;for(var s=2;s{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>a,metadata:()=>p,toc:()=>s});var r=n(7462),o=(n(7294),n(3905));const a={},i="Plugin",p={unversionedId:"concepts/plugin",id:"concepts/plugin",title:"Plugin",description:"Optimus can provide support for various data warehouses & any third party system to handle all the data transformations",source:"@site/docs/concepts/plugin.md",sourceDirName:"concepts",slug:"/concepts/plugin",permalink:"/optimus/docs/concepts/plugin",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/plugin.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Secret",permalink:"/optimus/docs/concepts/secret"},next:{title:"Replay & Backup",permalink:"/optimus/docs/concepts/replay-and-backup"}},c={},s=[],l={toc:s},u="wrapper";function d(e){let{components:t,...n}=e;return(0,o.kt)(u,(0,r.Z)({},l,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"plugin"},"Plugin"),(0,o.kt)("p",null,"Optimus can provide support for various data warehouses & any third party system to handle all the data transformations\nor movement through plugins. You can bring your own plugin by encapsulating all the logic in a docker container."),(0,o.kt)("p",null,"Currently, plugins can be defined as YAML or binary executables. YAML plugin provides the questionnaire and default\nvalues for job task\u2019s / hook\u2019s creation, as well as defines the image to execute. While a binary plugin, it is\ncomplementing the YAML plugin by providing support for automated dependency resolution."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9861],{3905:(e,t,n)=>{n.d(t,{Zo:()=>l,kt:()=>f});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=r.createContext({}),s=function(e){var t=r.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},l=function(e){var t=s(e.components);return r.createElement(c.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,c=e.parentName,l=p(e,["components","mdxType","originalType","parentName"]),u=s(n),m=o,f=u["".concat(c,".").concat(m)]||u[m]||d[m]||a;return n?r.createElement(f,i(i({ref:t},l),{},{components:n})):r.createElement(f,i({ref:t},l))}));function f(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,i=new Array(a);i[0]=m;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p[u]="string"==typeof e?e:o,i[1]=p;for(var s=2;s{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>a,metadata:()=>p,toc:()=>s});var r=n(7462),o=(n(7294),n(3905));const a={},i="Plugin",p={unversionedId:"concepts/plugin",id:"concepts/plugin",title:"Plugin",description:"Optimus can provide support for various data warehouses & any third party system to handle all the data transformations",source:"@site/docs/concepts/plugin.md",sourceDirName:"concepts",slug:"/concepts/plugin",permalink:"/optimus/docs/concepts/plugin",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/plugin.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Secret",permalink:"/optimus/docs/concepts/secret"},next:{title:"Replay & Backup",permalink:"/optimus/docs/concepts/replay-and-backup"}},c={},s=[],l={toc:s},u="wrapper";function d(e){let{components:t,...n}=e;return(0,o.kt)(u,(0,r.Z)({},l,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"plugin"},"Plugin"),(0,o.kt)("p",null,"Optimus can provide support for various data warehouses & any third party system to handle all the data transformations\nor movement through plugins. You can bring your own plugin by encapsulating all the logic in a docker container."),(0,o.kt)("p",null,"Currently, plugins can be defined as YAML or binary executables. YAML plugin provides the questionnaire and default\nvalues for job task\u2019s / hook\u2019s creation, as well as defines the image to execute. While a binary plugin, it is\ncomplementing the YAML plugin by providing support for automated dependency resolution."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/8a1416ba.a6a96efb.js b/assets/js/8a1416ba.eb20bc0d.js similarity index 98% rename from assets/js/8a1416ba.a6a96efb.js rename to assets/js/8a1416ba.eb20bc0d.js index c96988c277..26cb139b72 100644 --- a/assets/js/8a1416ba.a6a96efb.js +++ b/assets/js/8a1416ba.eb20bc0d.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6886],{3905:(e,t,r)=>{r.d(t,{Zo:()=>u,kt:()=>m});var n=r(7294);function i(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function o(e){for(var t=1;t=0||(i[r]=e[r]);return i}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(i[r]=e[r])}return i}var c=n.createContext({}),l=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):o(o({},t),e)),r},u=function(e){var t=l(e.components);return n.createElement(c.Provider,{value:t},e.children)},p="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},f=n.forwardRef((function(e,t){var r=e.components,i=e.mdxType,a=e.originalType,c=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),p=l(r),f=i,m=p["".concat(c,".").concat(f)]||p[f]||d[f]||a;return r?n.createElement(m,o(o({ref:t},u),{},{components:r})):n.createElement(m,o({ref:t},u))}));function m(e,t){var r=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var a=r.length,o=new Array(a);o[0]=f;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s[p]="string"==typeof e?e:i,o[1]=s;for(var l=2;l{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>o,default:()=>d,frontMatter:()=>a,metadata:()=>s,toc:()=>l});var n=r(7462),i=(r(7294),r(3905));const a={},o="Architecture",s={unversionedId:"concepts/architecture",id:"concepts/architecture",title:"Architecture",description:"Architecture Diagram",source:"@site/docs/concepts/architecture.md",sourceDirName:"concepts",slug:"/concepts/architecture",permalink:"/optimus/docs/concepts/architecture",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/architecture.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Quickstart",permalink:"/optimus/docs/getting-started/quick-start"},next:{title:"Project",permalink:"/optimus/docs/concepts/project"}},c={},l=[{value:"CLI",id:"cli",level:2},{value:"Server",id:"server",level:2},{value:"Database",id:"database",level:2},{value:"Plugins",id:"plugins",level:2},{value:"Scheduler (Airflow)",id:"scheduler-airflow",level:2}],u={toc:l},p="wrapper";function d(e){let{components:t,...a}=e;return(0,i.kt)(p,(0,n.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"architecture"},"Architecture"),(0,i.kt)("p",null,(0,i.kt)("img",{alt:"Architecture Diagram",src:r(3998).Z,title:"OptimusArchitecture",width:"1600",height:"1202"})),(0,i.kt)("h2",{id:"cli"},"CLI"),(0,i.kt)("p",null,"Optimus provides a command line interface to interact with the main optimus service and basic scaffolding job\nspecifications. It can be used to:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Start optimus server"),(0,i.kt)("li",{parentName:"ul"},"Create resource specifications for datastores"),(0,i.kt)("li",{parentName:"ul"},"Generate jobs & hooks based on user inputs"),(0,i.kt)("li",{parentName:"ul"},"Dump a compiled specification for the consumption of a scheduler"),(0,i.kt)("li",{parentName:"ul"},"Validate and inspect job specifications"),(0,i.kt)("li",{parentName:"ul"},"Deployment of specifications to Optimus Service")),(0,i.kt)("h2",{id:"server"},"Server"),(0,i.kt)("p",null,"Optimus Server handles all the client requests from direct end users or from airflow over http & grpc. The functionality\nof the server can be extended with the support of various plugins to various data sources & sinks. Everything around\njob/resource management is handled by the server except scheduling of jobs."),(0,i.kt)("h2",{id:"database"},"Database"),(0,i.kt)("p",null,"Optimus supports postgres as the main storage backend. It is the source of truth for all user specifications,\nconfigurations, secrets, assets. It is the place where all the precomputed relations between jobs are stored."),(0,i.kt)("h2",{id:"plugins"},"Plugins"),(0,i.kt)("p",null,"Currently, Optimus doesn\u2019t hold any logic and is not responsible for handling any specific transformations. This\ncapability is extended through plugins and users can customize based on their needs on what plugins to use. Plugins can\nbe defined through a yaml specification. At the time of execution whatever the image that is configured in the plugin\nimage will be executed."),(0,i.kt)("h2",{id:"scheduler-airflow"},"Scheduler (Airflow)"),(0,i.kt)("p",null,"Scheduler is responsible for scheduling all the user defined jobs. Currently, optimus supports only Airflow as\nthe scheduler, support for more schedulers can be added."))}d.isMDXComponent=!0},3998:(e,t,r)=>{r.d(t,{Z:()=>n});const n=r.p+"assets/images/OptimusArchitecture-8c62094985b762ca9f1848f4976fdfee.png"}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6886],{3905:(e,t,r)=>{r.d(t,{Zo:()=>u,kt:()=>m});var n=r(7294);function i(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function o(e){for(var t=1;t=0||(i[r]=e[r]);return i}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(i[r]=e[r])}return i}var c=n.createContext({}),l=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):o(o({},t),e)),r},u=function(e){var t=l(e.components);return n.createElement(c.Provider,{value:t},e.children)},p="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},f=n.forwardRef((function(e,t){var r=e.components,i=e.mdxType,a=e.originalType,c=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),p=l(r),f=i,m=p["".concat(c,".").concat(f)]||p[f]||d[f]||a;return r?n.createElement(m,o(o({ref:t},u),{},{components:r})):n.createElement(m,o({ref:t},u))}));function m(e,t){var r=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var a=r.length,o=new Array(a);o[0]=f;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s[p]="string"==typeof e?e:i,o[1]=s;for(var l=2;l{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>o,default:()=>d,frontMatter:()=>a,metadata:()=>s,toc:()=>l});var n=r(7462),i=(r(7294),r(3905));const a={},o="Architecture",s={unversionedId:"concepts/architecture",id:"concepts/architecture",title:"Architecture",description:"Architecture Diagram",source:"@site/docs/concepts/architecture.md",sourceDirName:"concepts",slug:"/concepts/architecture",permalink:"/optimus/docs/concepts/architecture",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/architecture.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Quickstart",permalink:"/optimus/docs/getting-started/quick-start"},next:{title:"Project",permalink:"/optimus/docs/concepts/project"}},c={},l=[{value:"CLI",id:"cli",level:2},{value:"Server",id:"server",level:2},{value:"Database",id:"database",level:2},{value:"Plugins",id:"plugins",level:2},{value:"Scheduler (Airflow)",id:"scheduler-airflow",level:2}],u={toc:l},p="wrapper";function d(e){let{components:t,...a}=e;return(0,i.kt)(p,(0,n.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"architecture"},"Architecture"),(0,i.kt)("p",null,(0,i.kt)("img",{alt:"Architecture Diagram",src:r(3998).Z,title:"OptimusArchitecture",width:"1600",height:"1202"})),(0,i.kt)("h2",{id:"cli"},"CLI"),(0,i.kt)("p",null,"Optimus provides a command line interface to interact with the main optimus service and basic scaffolding job\nspecifications. It can be used to:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Start optimus server"),(0,i.kt)("li",{parentName:"ul"},"Create resource specifications for datastores"),(0,i.kt)("li",{parentName:"ul"},"Generate jobs & hooks based on user inputs"),(0,i.kt)("li",{parentName:"ul"},"Dump a compiled specification for the consumption of a scheduler"),(0,i.kt)("li",{parentName:"ul"},"Validate and inspect job specifications"),(0,i.kt)("li",{parentName:"ul"},"Deployment of specifications to Optimus Service")),(0,i.kt)("h2",{id:"server"},"Server"),(0,i.kt)("p",null,"Optimus Server handles all the client requests from direct end users or from airflow over http & grpc. The functionality\nof the server can be extended with the support of various plugins to various data sources & sinks. Everything around\njob/resource management is handled by the server except scheduling of jobs."),(0,i.kt)("h2",{id:"database"},"Database"),(0,i.kt)("p",null,"Optimus supports postgres as the main storage backend. It is the source of truth for all user specifications,\nconfigurations, secrets, assets. It is the place where all the precomputed relations between jobs are stored."),(0,i.kt)("h2",{id:"plugins"},"Plugins"),(0,i.kt)("p",null,"Currently, Optimus doesn\u2019t hold any logic and is not responsible for handling any specific transformations. This\ncapability is extended through plugins and users can customize based on their needs on what plugins to use. Plugins can\nbe defined through a yaml specification. At the time of execution whatever the image that is configured in the plugin\nimage will be executed."),(0,i.kt)("h2",{id:"scheduler-airflow"},"Scheduler (Airflow)"),(0,i.kt)("p",null,"Scheduler is responsible for scheduling all the user defined jobs. Currently, optimus supports only Airflow as\nthe scheduler, support for more schedulers can be added."))}d.isMDXComponent=!0},3998:(e,t,r)=>{r.d(t,{Z:()=>n});const n=r.p+"assets/images/OptimusArchitecture-8c62094985b762ca9f1848f4976fdfee.png"}}]); \ No newline at end of file diff --git a/assets/js/8ce98423.33350de4.js b/assets/js/8ce98423.3a8598d4.js similarity index 99% rename from assets/js/8ce98423.33350de4.js rename to assets/js/8ce98423.3a8598d4.js index 325ae71a46..50e3a6ce1a 100644 --- a/assets/js/8ce98423.33350de4.js +++ b/assets/js/8ce98423.3a8598d4.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9987],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>g});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var s=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=r.createContext({}),c=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=c(e.components);return r.createElement(l.Provider,{value:t},e.children)},p="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,s=e.originalType,l=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),p=c(n),m=a,g=p["".concat(l,".").concat(m)]||p[m]||d[m]||s;return n?r.createElement(g,i(i({ref:t},u),{},{components:n})):r.createElement(g,i({ref:t},u))}));function g(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var s=n.length,i=new Array(s);i[0]=m;var o={};for(var l in t)hasOwnProperty.call(t,l)&&(o[l]=t[l]);o.originalType=e,o[p]="string"==typeof e?e:a,i[1]=o;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>i,default:()=>d,frontMatter:()=>s,metadata:()=>o,toc:()=>c});var r=n(7462),a=(n(7294),n(3905));const s={},i="Managing Secrets",o={unversionedId:"client-guide/managing-secrets",id:"client-guide/managing-secrets",title:"Managing Secrets",description:"During job execution, specific credentials are needed to access required resources, for example, BigQuery credential",source:"@site/docs/client-guide/managing-secrets.md",sourceDirName:"client-guide",slug:"/client-guide/managing-secrets",permalink:"/optimus/docs/client-guide/managing-secrets",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/managing-secrets.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Managing Project & Namespace",permalink:"/optimus/docs/client-guide/managing-project-namespace"},next:{title:"Installing Plugin in Client",permalink:"/optimus/docs/client-guide/installing-plugin"}},l={},c=[{value:"Registering secret",id:"registering-secret",level:2},{value:"Updating a secret",id:"updating-a-secret",level:2},{value:"Listing secrets",id:"listing-secrets",level:2}],u={toc:c},p="wrapper";function d(e){let{components:t,...n}=e;return(0,a.kt)(p,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"managing-secrets"},"Managing Secrets"),(0,a.kt)("p",null,"During job execution, specific credentials are needed to access required resources, for example, BigQuery credential\nfor BQ to BQ tasks. Users are able to register secrets on their own, manage them, and use them in tasks and hooks.\nPlease go through ",(0,a.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/secret"},"concepts")," to know more about secrets."),(0,a.kt)("p",null,"Before we begin, let\u2019s take a look at several mandatory secrets that is used for specific use cases in Optimus."),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Secret Name"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"STORAGE"),(0,a.kt)("td",{parentName:"tr",align:null},"To store compiled jobs if needed.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"SCHEDULER_AUTH"),(0,a.kt)("td",{parentName:"tr",align:null},"Scheduler credentials. For now, since Optimus only supports Airflow, this will be Airflow ","[username:password]")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"BQ_SERVICE_ACCOUNT"),(0,a.kt)("td",{parentName:"tr",align:null},"Used for any operations involving BigQuery, such as job validation, deployment, run for jobs with BQ to BQ transformation task, as well as for managing BigQuery resources through Optimus.")))),(0,a.kt)("h2",{id:"registering-secret"},"Registering secret"),(0,a.kt)("p",null,"Register a secret by running the following command:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret someSecretValue\n")),(0,a.kt)("p",null,"By default, Optimus will encode the secret value. However, to register a secret that has been encoded, run the following\ncommand instead:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret encodedSecretValue --base64\n")),(0,a.kt)("p",null,"There is also a flexibility to register using an existing secret file, instead of providing the secret value in the command."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret --file=/path/to/secret\n")),(0,a.kt)("p",null,"Secret can also be set to a specific namespace which can only be used by the jobs/resources in the namespace.\nTo register, run the following command:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret someSecretValue --namespace someNamespace\n")),(0,a.kt)("p",null,"Please note that registering a secret that already exists will result in an error. Modifying an existing secret\ncan be done using the Update command."),(0,a.kt)("h2",{id:"updating-a-secret"},"Updating a secret"),(0,a.kt)("p",null,"The update-only flag is generally used when you explicitly only want to update a secret that already exists and doesn't want to create it by mistake."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret someSecretValue --update-only\n")),(0,a.kt)("p",null,"It will return an error if the secret to update does not exist already."),(0,a.kt)("h2",{id:"listing-secrets"},"Listing secrets"),(0,a.kt)("p",null,"The list command can be used to show the user-defined secrets which are registered with Optimus. It will list the namespace associated with a secret."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret list\nSecrets for project: optimus-local\nNAME | DIGEST | NAMESPACE | DATE\n-------------+----------------------------------------------+-----------+----------------------\nsecret1 | SIBzsgUuHnExBY4qSzqcrlrb+3zCAHGu/4Fv1O8eMI8= | * | 2022-04-12T04:30:45Z\n")),(0,a.kt)("p",null,"It shows a digest for the encrypted secret, so as not to send the cleartext password on the network."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9987],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>g});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var s=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=r.createContext({}),c=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=c(e.components);return r.createElement(l.Provider,{value:t},e.children)},p="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,s=e.originalType,l=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),p=c(n),m=a,g=p["".concat(l,".").concat(m)]||p[m]||d[m]||s;return n?r.createElement(g,i(i({ref:t},u),{},{components:n})):r.createElement(g,i({ref:t},u))}));function g(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var s=n.length,i=new Array(s);i[0]=m;var o={};for(var l in t)hasOwnProperty.call(t,l)&&(o[l]=t[l]);o.originalType=e,o[p]="string"==typeof e?e:a,i[1]=o;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>i,default:()=>d,frontMatter:()=>s,metadata:()=>o,toc:()=>c});var r=n(7462),a=(n(7294),n(3905));const s={},i="Managing Secrets",o={unversionedId:"client-guide/managing-secrets",id:"client-guide/managing-secrets",title:"Managing Secrets",description:"During job execution, specific credentials are needed to access required resources, for example, BigQuery credential",source:"@site/docs/client-guide/managing-secrets.md",sourceDirName:"client-guide",slug:"/client-guide/managing-secrets",permalink:"/optimus/docs/client-guide/managing-secrets",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/managing-secrets.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Managing Project & Namespace",permalink:"/optimus/docs/client-guide/managing-project-namespace"},next:{title:"Installing Plugin in Client",permalink:"/optimus/docs/client-guide/installing-plugin"}},l={},c=[{value:"Registering secret",id:"registering-secret",level:2},{value:"Updating a secret",id:"updating-a-secret",level:2},{value:"Listing secrets",id:"listing-secrets",level:2}],u={toc:c},p="wrapper";function d(e){let{components:t,...n}=e;return(0,a.kt)(p,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"managing-secrets"},"Managing Secrets"),(0,a.kt)("p",null,"During job execution, specific credentials are needed to access required resources, for example, BigQuery credential\nfor BQ to BQ tasks. Users are able to register secrets on their own, manage them, and use them in tasks and hooks.\nPlease go through ",(0,a.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/secret"},"concepts")," to know more about secrets."),(0,a.kt)("p",null,"Before we begin, let\u2019s take a look at several mandatory secrets that is used for specific use cases in Optimus."),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Secret Name"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"STORAGE"),(0,a.kt)("td",{parentName:"tr",align:null},"To store compiled jobs if needed.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"SCHEDULER_AUTH"),(0,a.kt)("td",{parentName:"tr",align:null},"Scheduler credentials. For now, since Optimus only supports Airflow, this will be Airflow ","[username:password]")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"BQ_SERVICE_ACCOUNT"),(0,a.kt)("td",{parentName:"tr",align:null},"Used for any operations involving BigQuery, such as job validation, deployment, run for jobs with BQ to BQ transformation task, as well as for managing BigQuery resources through Optimus.")))),(0,a.kt)("h2",{id:"registering-secret"},"Registering secret"),(0,a.kt)("p",null,"Register a secret by running the following command:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret someSecretValue\n")),(0,a.kt)("p",null,"By default, Optimus will encode the secret value. However, to register a secret that has been encoded, run the following\ncommand instead:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret encodedSecretValue --base64\n")),(0,a.kt)("p",null,"There is also a flexibility to register using an existing secret file, instead of providing the secret value in the command."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret --file=/path/to/secret\n")),(0,a.kt)("p",null,"Secret can also be set to a specific namespace which can only be used by the jobs/resources in the namespace.\nTo register, run the following command:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret someSecretValue --namespace someNamespace\n")),(0,a.kt)("p",null,"Please note that registering a secret that already exists will result in an error. Modifying an existing secret\ncan be done using the Update command."),(0,a.kt)("h2",{id:"updating-a-secret"},"Updating a secret"),(0,a.kt)("p",null,"The update-only flag is generally used when you explicitly only want to update a secret that already exists and doesn't want to create it by mistake."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret set someSecret someSecretValue --update-only\n")),(0,a.kt)("p",null,"It will return an error if the secret to update does not exist already."),(0,a.kt)("h2",{id:"listing-secrets"},"Listing secrets"),(0,a.kt)("p",null,"The list command can be used to show the user-defined secrets which are registered with Optimus. It will list the namespace associated with a secret."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus secret list\nSecrets for project: optimus-local\nNAME | DIGEST | NAMESPACE | DATE\n-------------+----------------------------------------------+-----------+----------------------\nsecret1 | SIBzsgUuHnExBY4qSzqcrlrb+3zCAHGu/4Fv1O8eMI8= | * | 2022-04-12T04:30:45Z\n")),(0,a.kt)("p",null,"It shows a digest for the encrypted secret, so as not to send the cleartext password on the network."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/9efad6bf.73405ccb.js b/assets/js/9efad6bf.ba1cc1b9.js similarity index 98% rename from assets/js/9efad6bf.73405ccb.js rename to assets/js/9efad6bf.ba1cc1b9.js index a6d852e48d..99772194b4 100644 --- a/assets/js/9efad6bf.73405ccb.js +++ b/assets/js/9efad6bf.ba1cc1b9.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6223],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>m});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=r.createContext({}),s=function(e){var t=r.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=s(e.components);return r.createElement(c.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},f=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,c=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=s(n),f=a,m=u["".concat(c,".").concat(f)]||u[f]||d[f]||o;return n?r.createElement(m,i(i({ref:t},p),{},{components:n})):r.createElement(m,i({ref:t},p))}));function m(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,i=new Array(o);i[0]=f;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l[u]="string"==typeof e?e:a,i[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>o,metadata:()=>l,toc:()=>s});var r=n(7462),a=(n(7294),n(3905));const o={},i="Replay & Backup",l={unversionedId:"concepts/replay-and-backup",id:"concepts/replay-and-backup",title:"Replay & Backup",description:"A job might need to be re-run (backfill) due to business requirement changes or other various reasons. Optimus provides",source:"@site/docs/concepts/replay-and-backup.md",sourceDirName:"concepts",slug:"/concepts/replay-and-backup",permalink:"/optimus/docs/concepts/replay-and-backup",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/replay-and-backup.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Plugin",permalink:"/optimus/docs/concepts/plugin"},next:{title:"Server Configuration",permalink:"/optimus/docs/server-guide/configuration"}},c={},s=[],p={toc:s},u="wrapper";function d(e){let{components:t,...o}=e;return(0,a.kt)(u,(0,r.Z)({},p,o,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"replay--backup"},"Replay & Backup"),(0,a.kt)("p",null,"A job might need to be re-run (backfill) due to business requirement changes or other various reasons. Optimus provides\nan easy way to do this using Replay. Replay accepts which job and range of date to be updated, validates it, and re-runs\nthe job tasks."),(0,a.kt)("p",null,"When validating, Optimus checks if there is any Replay with the same job and date currently running and also checks if\nthe task scheduler instances are still running to avoid any duplication and conflicts."),(0,a.kt)("p",null,"After passing the validation checks, a Replay request will be created and will be processed by the workers based on the\nmode chosen (sequential/parallel). To re-run the tasks, Optimus clears the existing runs from the scheduler."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Sequential (Default)")),(0,a.kt)("p",null,(0,a.kt)("img",{alt:"Sequential Mode Flow",src:n(9800).Z,title:"SequentialMode",width:"922",height:"222"})),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Parallel")),(0,a.kt)("p",null,(0,a.kt)("img",{alt:"Parallel Mode Flow",src:n(4097).Z,title:"ParallelMode",width:"814",height:"191"})),(0,a.kt)("p",null,"Optimus also provides a Backup feature to duplicate a resource that can be perfectly used before running Replay. Where\nthe backup result will be located, and the expiry detail can be configured in the project configuration."))}d.isMDXComponent=!0},4097:(e,t,n)=>{n.d(t,{Z:()=>r});const r=n.p+"assets/images/ReplayParallel-380cc531902871dd09bad2213c78b905.png"},9800:(e,t,n)=>{n.d(t,{Z:()=>r});const r=n.p+"assets/images/ReplaySequential-fc2c15aa94e0cdbde7348b508019546d.png"}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6223],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>m});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var c=r.createContext({}),s=function(e){var t=r.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=s(e.components);return r.createElement(c.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},f=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,c=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=s(n),f=a,m=u["".concat(c,".").concat(f)]||u[f]||d[f]||o;return n?r.createElement(m,i(i({ref:t},p),{},{components:n})):r.createElement(m,i({ref:t},p))}));function m(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,i=new Array(o);i[0]=f;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l[u]="string"==typeof e?e:a,i[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>d,frontMatter:()=>o,metadata:()=>l,toc:()=>s});var r=n(7462),a=(n(7294),n(3905));const o={},i="Replay & Backup",l={unversionedId:"concepts/replay-and-backup",id:"concepts/replay-and-backup",title:"Replay & Backup",description:"A job might need to be re-run (backfill) due to business requirement changes or other various reasons. Optimus provides",source:"@site/docs/concepts/replay-and-backup.md",sourceDirName:"concepts",slug:"/concepts/replay-and-backup",permalink:"/optimus/docs/concepts/replay-and-backup",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/replay-and-backup.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Plugin",permalink:"/optimus/docs/concepts/plugin"},next:{title:"Server Configuration",permalink:"/optimus/docs/server-guide/configuration"}},c={},s=[],p={toc:s},u="wrapper";function d(e){let{components:t,...o}=e;return(0,a.kt)(u,(0,r.Z)({},p,o,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"replay--backup"},"Replay & Backup"),(0,a.kt)("p",null,"A job might need to be re-run (backfill) due to business requirement changes or other various reasons. Optimus provides\nan easy way to do this using Replay. Replay accepts which job and range of date to be updated, validates it, and re-runs\nthe job tasks."),(0,a.kt)("p",null,"When validating, Optimus checks if there is any Replay with the same job and date currently running and also checks if\nthe task scheduler instances are still running to avoid any duplication and conflicts."),(0,a.kt)("p",null,"After passing the validation checks, a Replay request will be created and will be processed by the workers based on the\nmode chosen (sequential/parallel). To re-run the tasks, Optimus clears the existing runs from the scheduler."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Sequential (Default)")),(0,a.kt)("p",null,(0,a.kt)("img",{alt:"Sequential Mode Flow",src:n(9800).Z,title:"SequentialMode",width:"922",height:"222"})),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Parallel")),(0,a.kt)("p",null,(0,a.kt)("img",{alt:"Parallel Mode Flow",src:n(4097).Z,title:"ParallelMode",width:"814",height:"191"})),(0,a.kt)("p",null,"Optimus also provides a Backup feature to duplicate a resource that can be perfectly used before running Replay. Where\nthe backup result will be located, and the expiry detail can be configured in the project configuration."))}d.isMDXComponent=!0},4097:(e,t,n)=>{n.d(t,{Z:()=>r});const r=n.p+"assets/images/ReplayParallel-380cc531902871dd09bad2213c78b905.png"},9800:(e,t,n)=>{n.d(t,{Z:()=>r});const r=n.p+"assets/images/ReplaySequential-fc2c15aa94e0cdbde7348b508019546d.png"}}]); \ No newline at end of file diff --git a/assets/js/a09c2993.f0ff2f69.js b/assets/js/a09c2993.804b23b7.js similarity index 99% rename from assets/js/a09c2993.f0ff2f69.js rename to assets/js/a09c2993.804b23b7.js index b3e701309e..bc3f0711a7 100644 --- a/assets/js/a09c2993.f0ff2f69.js +++ b/assets/js/a09c2993.804b23b7.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[4128],{3905:(e,t,n)=>{n.d(t,{Zo:()=>c,kt:()=>h});var i=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function a(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var u=i.createContext({}),l=function(e){var t=i.useContext(u),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},c=function(e){var t=l(e.components);return i.createElement(u.Provider,{value:t},e.children)},p="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},m=i.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,u=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=l(n),m=o,h=p["".concat(u,".").concat(m)]||p[m]||d[m]||r;return n?i.createElement(h,a(a({ref:t},c),{},{components:n})):i.createElement(h,a({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,a=new Array(r);a[0]=m;var s={};for(var u in t)hasOwnProperty.call(t,u)&&(s[u]=t[u]);s.originalType=e,s[p]="string"==typeof e?e:o,a[1]=s;for(var l=2;l{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>a,default:()=>d,frontMatter:()=>r,metadata:()=>s,toc:()=>l});var i=n(7462),o=(n(7294),n(3905));const r={id:"introduction",title:"Introduction"},a="Optimus",s={unversionedId:"introduction",id:"introduction",title:"Introduction",description:"Optimus is an ETL orchestration tool that helps manage data transformation jobs and manage warehouse resources.",source:"@site/docs/introduction.md",sourceDirName:".",slug:"/introduction",permalink:"/optimus/docs/introduction",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/introduction.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{id:"introduction",title:"Introduction"},sidebar:"docsSidebar",next:{title:"Installation",permalink:"/optimus/docs/getting-started/installation"}},u={},l=[{value:"Multi-Tenancy Support",id:"multi-tenancy-support",level:2},{value:"Extensible",id:"extensible",level:2},{value:"Automated Dependency Resolution",id:"automated-dependency-resolution",level:2},{value:"In-Built Alerting",id:"in-built-alerting",level:2},{value:"Verification in Advance",id:"verification-in-advance",level:2}],c={toc:l},p="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(p,(0,i.Z)({},c,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"optimus"},"Optimus"),(0,o.kt)("p",null,"Optimus is an ETL orchestration tool that helps manage data transformation jobs and manage warehouse resources.\nIt enables you to transform your data by writing the transformation script and YAML configuration while Optimus handles\nthe dependency, schedules it, and handles all other aspects of running transformation jobs at scale. Optimus also supports\nwarehouse resource management (currently BigQuery), which enables you to create, update, and read BigQuery tables, views, and datasets."),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"High Level Optimus Diagram",src:n(2273).Z,title:"OptimusIntro",width:"1600",height:"745"})),(0,o.kt)("p",null,"Optimus was made to be extensible. Adding support for different kinds of sources/sinks and transformation executors\ncan be done easily. If your organization has to setup & manage data pipelines that are complex with multiple sources,\nsinks & there are many team members managing them, then Optimus is the perfect tool for you."),(0,o.kt)("h2",{id:"multi-tenancy-support"},"Multi-Tenancy Support"),(0,o.kt)("p",null,"Optimus supports multi-tenancy. Each tenant manages their own jobs, resources, secrets, and configuration while optimus\nmanaging dependencies across tenants."),(0,o.kt)("h2",{id:"extensible"},"Extensible"),(0,o.kt)("p",null,"Optimus provides the flexibility to you to define how your transformation jobs should behave, which data source or\nwarehouse sink you want to support, and what configurations you need from the users. This flexibility is addressed\nthrough ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/plugin"},"plugin"),". At the moment, we provide a ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/transformers/tree/main/task/bq2bq"},"BigQuery to BigQuery task plugin"),",\nbut you can write custom plugins such as Python transformations."),(0,o.kt)("p",null,"Also, in order to provide a unified command line experience of various tools, Optimus provides ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/work-with-extension"},"extensions"),"\nsupport on client side through which you can extend the capabilities for example providing governance. "),(0,o.kt)("h2",{id:"automated-dependency-resolution"},"Automated Dependency Resolution"),(0,o.kt)("p",null,"Optimus parses your data transformation queries and builds a dependency graph automatically without the user explicitly\ndefining the same. The dependencies are managed across tenants, so teams doesn\u2019t need to coordinate among themselves."),(0,o.kt)("h2",{id:"in-built-alerting"},"In-Built Alerting"),(0,o.kt)("p",null,"Always get notified when your job is not behaving as expected, on failures or on SLA misses. Optimus supports\nintegrations with slack & pagerduty."),(0,o.kt)("h2",{id:"verification-in-advance"},"Verification in Advance"),(0,o.kt)("p",null,"Minimize job runtime issues by validating and inspecting jobs before submitting them which enables them for faster\nturnaround time when submitting their jobs. Users can get to know about job dependencies, validation failures & some\nwarnings before submitting the jobs."))}d.isMDXComponent=!0},2273:(e,t,n)=>{n.d(t,{Z:()=>i});const i=n.p+"assets/images/OptimusIntro-0990771857ec459389c809e9c874e71a.png"}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[4128],{3905:(e,t,n)=>{n.d(t,{Zo:()=>c,kt:()=>h});var i=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function a(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var u=i.createContext({}),l=function(e){var t=i.useContext(u),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},c=function(e){var t=l(e.components);return i.createElement(u.Provider,{value:t},e.children)},p="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},m=i.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,u=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),p=l(n),m=o,h=p["".concat(u,".").concat(m)]||p[m]||d[m]||r;return n?i.createElement(h,a(a({ref:t},c),{},{components:n})):i.createElement(h,a({ref:t},c))}));function h(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,a=new Array(r);a[0]=m;var s={};for(var u in t)hasOwnProperty.call(t,u)&&(s[u]=t[u]);s.originalType=e,s[p]="string"==typeof e?e:o,a[1]=s;for(var l=2;l{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>a,default:()=>d,frontMatter:()=>r,metadata:()=>s,toc:()=>l});var i=n(7462),o=(n(7294),n(3905));const r={id:"introduction",title:"Introduction"},a="Optimus",s={unversionedId:"introduction",id:"introduction",title:"Introduction",description:"Optimus is an ETL orchestration tool that helps manage data transformation jobs and manage warehouse resources.",source:"@site/docs/introduction.md",sourceDirName:".",slug:"/introduction",permalink:"/optimus/docs/introduction",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/introduction.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{id:"introduction",title:"Introduction"},sidebar:"docsSidebar",next:{title:"Installation",permalink:"/optimus/docs/getting-started/installation"}},u={},l=[{value:"Multi-Tenancy Support",id:"multi-tenancy-support",level:2},{value:"Extensible",id:"extensible",level:2},{value:"Automated Dependency Resolution",id:"automated-dependency-resolution",level:2},{value:"In-Built Alerting",id:"in-built-alerting",level:2},{value:"Verification in Advance",id:"verification-in-advance",level:2}],c={toc:l},p="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(p,(0,i.Z)({},c,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"optimus"},"Optimus"),(0,o.kt)("p",null,"Optimus is an ETL orchestration tool that helps manage data transformation jobs and manage warehouse resources.\nIt enables you to transform your data by writing the transformation script and YAML configuration while Optimus handles\nthe dependency, schedules it, and handles all other aspects of running transformation jobs at scale. Optimus also supports\nwarehouse resource management (currently BigQuery), which enables you to create, update, and read BigQuery tables, views, and datasets."),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"High Level Optimus Diagram",src:n(2273).Z,title:"OptimusIntro",width:"1600",height:"745"})),(0,o.kt)("p",null,"Optimus was made to be extensible. Adding support for different kinds of sources/sinks and transformation executors\ncan be done easily. If your organization has to setup & manage data pipelines that are complex with multiple sources,\nsinks & there are many team members managing them, then Optimus is the perfect tool for you."),(0,o.kt)("h2",{id:"multi-tenancy-support"},"Multi-Tenancy Support"),(0,o.kt)("p",null,"Optimus supports multi-tenancy. Each tenant manages their own jobs, resources, secrets, and configuration while optimus\nmanaging dependencies across tenants."),(0,o.kt)("h2",{id:"extensible"},"Extensible"),(0,o.kt)("p",null,"Optimus provides the flexibility to you to define how your transformation jobs should behave, which data source or\nwarehouse sink you want to support, and what configurations you need from the users. This flexibility is addressed\nthrough ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/plugin"},"plugin"),". At the moment, we provide a ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/transformers/tree/main/task/bq2bq"},"BigQuery to BigQuery task plugin"),",\nbut you can write custom plugins such as Python transformations."),(0,o.kt)("p",null,"Also, in order to provide a unified command line experience of various tools, Optimus provides ",(0,o.kt)("a",{parentName:"p",href:"/optimus/docs/client-guide/work-with-extension"},"extensions"),"\nsupport on client side through which you can extend the capabilities for example providing governance. "),(0,o.kt)("h2",{id:"automated-dependency-resolution"},"Automated Dependency Resolution"),(0,o.kt)("p",null,"Optimus parses your data transformation queries and builds a dependency graph automatically without the user explicitly\ndefining the same. The dependencies are managed across tenants, so teams doesn\u2019t need to coordinate among themselves."),(0,o.kt)("h2",{id:"in-built-alerting"},"In-Built Alerting"),(0,o.kt)("p",null,"Always get notified when your job is not behaving as expected, on failures or on SLA misses. Optimus supports\nintegrations with slack & pagerduty."),(0,o.kt)("h2",{id:"verification-in-advance"},"Verification in Advance"),(0,o.kt)("p",null,"Minimize job runtime issues by validating and inspecting jobs before submitting them which enables them for faster\nturnaround time when submitting their jobs. Users can get to know about job dependencies, validation failures & some\nwarnings before submitting the jobs."))}d.isMDXComponent=!0},2273:(e,t,n)=>{n.d(t,{Z:()=>i});const i=n.p+"assets/images/OptimusIntro-0990771857ec459389c809e9c874e71a.png"}}]); \ No newline at end of file diff --git a/assets/js/ab079b58.cca0df44.js b/assets/js/ab079b58.658a2062.js similarity index 99% rename from assets/js/ab079b58.cca0df44.js rename to assets/js/ab079b58.658a2062.js index 9476cd837c..b88002a4cd 100644 --- a/assets/js/ab079b58.cca0df44.js +++ b/assets/js/ab079b58.658a2062.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2994],{3905:(e,t,a)=>{a.d(t,{Zo:()=>c,kt:()=>f});var n=a(7294);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function l(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var i=n.createContext({}),p=function(e){var t=n.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},c=function(e){var t=p(e.components);return n.createElement(i.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(a),d=o,f=u["".concat(i,".").concat(d)]||u[d]||m[d]||r;return a?n.createElement(f,l(l({ref:t},c),{},{components:a})):n.createElement(f,l({ref:t},c))}));function f(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,l=new Array(r);l[0]=d;var s={};for(var i in t)hasOwnProperty.call(t,i)&&(s[i]=t[i]);s.originalType=e,s[u]="string"==typeof e?e:o,l[1]=s;for(var p=2;p{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>l,default:()=>m,frontMatter:()=>r,metadata:()=>s,toc:()=>p});var n=a(7462),o=(a(7294),a(3905));const r={},l="Job",s={unversionedId:"concepts/job",id:"concepts/job",title:"Job",description:"A Job is the fundamental unit of the data pipeline which enables a data transformation in the warehouse of choice.",source:"@site/docs/concepts/job.md",sourceDirName:"concepts",slug:"/concepts/job",permalink:"/optimus/docs/concepts/job",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/job.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Resource",permalink:"/optimus/docs/concepts/resource"},next:{title:"Job Run",permalink:"/optimus/docs/concepts/job-run"}},i={},p=[{value:"Task",id:"task",level:2},{value:"Hook",id:"hook",level:2},{value:"Asset",id:"asset",level:2}],c={toc:p},u="wrapper";function m(e){let{components:t,...a}=e;return(0,o.kt)(u,(0,n.Z)({},c,a,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"job"},"Job"),(0,o.kt)("p",null,"A Job is the fundamental unit of the data pipeline which enables a data transformation in the warehouse of choice.\nA user can configure various details mentioned below for the job:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Schedule interval"),(0,o.kt)("li",{parentName:"ul"},"Date from when a transformation should start executing"),(0,o.kt)("li",{parentName:"ul"},"Task & Hooks"),(0,o.kt)("li",{parentName:"ul"},"Assets needed for transformation"),(0,o.kt)("li",{parentName:"ul"},"Alerts")),(0,o.kt)("p",null,"Job specifications are being compiled to later be processed by the scheduler. Optimus is using Airflow as the scheduler,\nthus it is compiling the job specification to DAG (",(0,o.kt)("em",{parentName:"p"},"Directed Acryclic Graph"),") file."),(0,o.kt)("p",null,"Each of the DAG represents a single job, which consists of:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Airflow task(s). Transformation tasks and hooks will be compiled to Airflow tasks."),(0,o.kt)("li",{parentName:"ul"},"Sensors, only if the job has dependency.")),(0,o.kt)("p",null,"Each job has a single base transformation, we call them ",(0,o.kt)("strong",{parentName:"p"},"Task")," and might have the task pre or/and post operations,\nwhich are called ",(0,o.kt)("strong",{parentName:"p"},"Hooks"),"."),(0,o.kt)("h2",{id:"task"},"Task"),(0,o.kt)("p",null,"A task is a main transformation process that will fetch data, transform as configured, and sink to the destination.\nEach task has its own set of configs and can inherit configurations from a global configuration store."),(0,o.kt)("p",null,"Some examples of task are:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"BQ to BQ task"),(0,o.kt)("li",{parentName:"ul"},"BQ to Email task"),(0,o.kt)("li",{parentName:"ul"},"Python task"),(0,o.kt)("li",{parentName:"ul"},"Tableau task"),(0,o.kt)("li",{parentName:"ul"},"Etc.")),(0,o.kt)("h2",{id:"hook"},"Hook"),(0,o.kt)("p",null,"Hooks are the operations that you might want to run before or after a task. A hook is only associated with a single\nparent although they can depend on other hooks within the same job. There can be one or many or zero hooks for a Job as\nconfigured by the user. Some examples of hooks are:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("a",{parentName:"li",href:"https://github.com/goto/predator"},"Predator")," (Profiling & Auditing for BQ)"),(0,o.kt)("li",{parentName:"ul"},"Publishing transformed data to Kafka"),(0,o.kt)("li",{parentName:"ul"},"Http Hooks")),(0,o.kt)("p",null,"Each hook has its own set of configs and shares the same asset folder as the base job. Hook can inherit configurations\nfrom the base transformation or from a global configuration store."),(0,o.kt)("p",null,"The fundamental difference between a hook and a task is, a task can have dependencies over other jobs inside the\nrepository whereas a hook can only depend on other hooks within the job."),(0,o.kt)("h2",{id:"asset"},"Asset"),(0,o.kt)("p",null,"There could be an asset folder along with the job.yaml file generated via optimus when a new job is created. This is a\nshared folder across base transformation task and all associated hooks. Assets can use macros and functions powered by\n",(0,o.kt)("a",{parentName:"p",href:"https://golang.org/pkg/text/template/"},"Go templating engine"),"."),(0,o.kt)("p",null,"Section of code can be imported from different asset files using template. For example:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"File partials.gtpl")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},"DECLARE t1 TIMESTAMP;\n")),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Another file query.sql")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},"{{template \"partials.gtpl\"}}\nSET t1 = '2021-02-10T10:00:00+00:00';\n")),(0,o.kt)("p",null,"During execution query.sql will be rendered as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},"DECLARE t1 TIMESTAMP;\nSET t1 = '2021-02-10T10:00:00+00:00';\n")),(0,o.kt)("p",null,"whereas ",(0,o.kt)("strong",{parentName:"p"},"partials.gtpl")," will be left as it is because file was saved with .gtpl extension."),(0,o.kt)("p",null,"Similarly, a single file can contain multiple blocks of code that can function as macro of code replacement. For example:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"file.data")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},'Name: {{ template "name"}}, Gender: {{ template "gender" }}\n')),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"partials.gtpl")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},'{{- define "name" -}} Adam {{- end}}\n{{- define "gender" -}} Male {{- end}}\n')),(0,o.kt)("p",null,"This will render file.data as"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"Name: Adam, Gender: Male\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2994],{3905:(e,t,a)=>{a.d(t,{Zo:()=>c,kt:()=>f});var n=a(7294);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function l(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var i=n.createContext({}),p=function(e){var t=n.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},c=function(e){var t=p(e.components);return n.createElement(i.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(a),d=o,f=u["".concat(i,".").concat(d)]||u[d]||m[d]||r;return a?n.createElement(f,l(l({ref:t},c),{},{components:a})):n.createElement(f,l({ref:t},c))}));function f(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,l=new Array(r);l[0]=d;var s={};for(var i in t)hasOwnProperty.call(t,i)&&(s[i]=t[i]);s.originalType=e,s[u]="string"==typeof e?e:o,l[1]=s;for(var p=2;p{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>l,default:()=>m,frontMatter:()=>r,metadata:()=>s,toc:()=>p});var n=a(7462),o=(a(7294),a(3905));const r={},l="Job",s={unversionedId:"concepts/job",id:"concepts/job",title:"Job",description:"A Job is the fundamental unit of the data pipeline which enables a data transformation in the warehouse of choice.",source:"@site/docs/concepts/job.md",sourceDirName:"concepts",slug:"/concepts/job",permalink:"/optimus/docs/concepts/job",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/job.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Resource",permalink:"/optimus/docs/concepts/resource"},next:{title:"Job Run",permalink:"/optimus/docs/concepts/job-run"}},i={},p=[{value:"Task",id:"task",level:2},{value:"Hook",id:"hook",level:2},{value:"Asset",id:"asset",level:2}],c={toc:p},u="wrapper";function m(e){let{components:t,...a}=e;return(0,o.kt)(u,(0,n.Z)({},c,a,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"job"},"Job"),(0,o.kt)("p",null,"A Job is the fundamental unit of the data pipeline which enables a data transformation in the warehouse of choice.\nA user can configure various details mentioned below for the job:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Schedule interval"),(0,o.kt)("li",{parentName:"ul"},"Date from when a transformation should start executing"),(0,o.kt)("li",{parentName:"ul"},"Task & Hooks"),(0,o.kt)("li",{parentName:"ul"},"Assets needed for transformation"),(0,o.kt)("li",{parentName:"ul"},"Alerts")),(0,o.kt)("p",null,"Job specifications are being compiled to later be processed by the scheduler. Optimus is using Airflow as the scheduler,\nthus it is compiling the job specification to DAG (",(0,o.kt)("em",{parentName:"p"},"Directed Acryclic Graph"),") file."),(0,o.kt)("p",null,"Each of the DAG represents a single job, which consists of:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Airflow task(s). Transformation tasks and hooks will be compiled to Airflow tasks."),(0,o.kt)("li",{parentName:"ul"},"Sensors, only if the job has dependency.")),(0,o.kt)("p",null,"Each job has a single base transformation, we call them ",(0,o.kt)("strong",{parentName:"p"},"Task")," and might have the task pre or/and post operations,\nwhich are called ",(0,o.kt)("strong",{parentName:"p"},"Hooks"),"."),(0,o.kt)("h2",{id:"task"},"Task"),(0,o.kt)("p",null,"A task is a main transformation process that will fetch data, transform as configured, and sink to the destination.\nEach task has its own set of configs and can inherit configurations from a global configuration store."),(0,o.kt)("p",null,"Some examples of task are:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"BQ to BQ task"),(0,o.kt)("li",{parentName:"ul"},"BQ to Email task"),(0,o.kt)("li",{parentName:"ul"},"Python task"),(0,o.kt)("li",{parentName:"ul"},"Tableau task"),(0,o.kt)("li",{parentName:"ul"},"Etc.")),(0,o.kt)("h2",{id:"hook"},"Hook"),(0,o.kt)("p",null,"Hooks are the operations that you might want to run before or after a task. A hook is only associated with a single\nparent although they can depend on other hooks within the same job. There can be one or many or zero hooks for a Job as\nconfigured by the user. Some examples of hooks are:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("a",{parentName:"li",href:"https://github.com/goto/predator"},"Predator")," (Profiling & Auditing for BQ)"),(0,o.kt)("li",{parentName:"ul"},"Publishing transformed data to Kafka"),(0,o.kt)("li",{parentName:"ul"},"Http Hooks")),(0,o.kt)("p",null,"Each hook has its own set of configs and shares the same asset folder as the base job. Hook can inherit configurations\nfrom the base transformation or from a global configuration store."),(0,o.kt)("p",null,"The fundamental difference between a hook and a task is, a task can have dependencies over other jobs inside the\nrepository whereas a hook can only depend on other hooks within the job."),(0,o.kt)("h2",{id:"asset"},"Asset"),(0,o.kt)("p",null,"There could be an asset folder along with the job.yaml file generated via optimus when a new job is created. This is a\nshared folder across base transformation task and all associated hooks. Assets can use macros and functions powered by\n",(0,o.kt)("a",{parentName:"p",href:"https://golang.org/pkg/text/template/"},"Go templating engine"),"."),(0,o.kt)("p",null,"Section of code can be imported from different asset files using template. For example:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"File partials.gtpl")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},"DECLARE t1 TIMESTAMP;\n")),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Another file query.sql")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},"{{template \"partials.gtpl\"}}\nSET t1 = '2021-02-10T10:00:00+00:00';\n")),(0,o.kt)("p",null,"During execution query.sql will be rendered as:"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},"DECLARE t1 TIMESTAMP;\nSET t1 = '2021-02-10T10:00:00+00:00';\n")),(0,o.kt)("p",null,"whereas ",(0,o.kt)("strong",{parentName:"p"},"partials.gtpl")," will be left as it is because file was saved with .gtpl extension."),(0,o.kt)("p",null,"Similarly, a single file can contain multiple blocks of code that can function as macro of code replacement. For example:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"file.data")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},'Name: {{ template "name"}}, Gender: {{ template "gender" }}\n')),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"partials.gtpl")),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-gotemplate"},'{{- define "name" -}} Adam {{- end}}\n{{- define "gender" -}} Male {{- end}}\n')),(0,o.kt)("p",null,"This will render file.data as"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre"},"Name: Adam, Gender: Male\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/ad92602d.f6d05e6f.js b/assets/js/ad92602d.436c4a20.js similarity index 99% rename from assets/js/ad92602d.f6d05e6f.js rename to assets/js/ad92602d.436c4a20.js index 438c4aa420..83db5bcf2c 100644 --- a/assets/js/ad92602d.f6d05e6f.js +++ b/assets/js/ad92602d.436c4a20.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6374],{3905:(e,n,t)=>{t.d(n,{Zo:()=>c,kt:()=>f});var a=t(7294);function r(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function i(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);n&&(a=a.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,a)}return t}function o(e){for(var n=1;n=0||(r[t]=e[t]);return r}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var l=a.createContext({}),p=function(e){var n=a.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):o(o({},n),e)),t},c=function(e){var n=p(e.components);return a.createElement(l.Provider,{value:n},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var n=e.children;return a.createElement(a.Fragment,{},n)}},m=a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(t),m=r,f=u["".concat(l,".").concat(m)]||u[m]||d[m]||i;return t?a.createElement(f,o(o({ref:n},c),{},{components:t})):a.createElement(f,o({ref:n},c))}));function f(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var i=t.length,o=new Array(i);o[0]=m;var s={};for(var l in n)hasOwnProperty.call(n,l)&&(s[l]=n[l]);s.originalType=e,s[u]="string"==typeof e?e:r,o[1]=s;for(var p=2;p{t.r(n),t.d(n,{assets:()=>l,contentTitle:()=>o,default:()=>d,frontMatter:()=>i,metadata:()=>s,toc:()=>p});var a=t(7462),r=(t(7294),t(3905));const i={},o="Organizing Specifications",s={unversionedId:"client-guide/organizing-specifications",id:"client-guide/organizing-specifications",title:"Organizing Specifications",description:"Optimus supports two ways to deploy specifications",source:"@site/docs/client-guide/organizing-specifications.md",sourceDirName:"client-guide",slug:"/client-guide/organizing-specifications",permalink:"/optimus/docs/client-guide/organizing-specifications",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/organizing-specifications.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Uploading Job to Scheduler",permalink:"/optimus/docs/client-guide/uploading-jobs-to-scheduler"},next:{title:"Backup BigQuery Resource",permalink:"/optimus/docs/client-guide/backup-bigquery-resource"}},l={},p=[],c={toc:p},u="wrapper";function d(e){let{components:n,...t}=e;return(0,r.kt)(u,(0,a.Z)({},c,t,{components:n,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"organizing-specifications"},"Organizing Specifications"),(0,r.kt)("p",null,"Optimus supports two ways to deploy specifications"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"REST/GRPC"),(0,r.kt)("li",{parentName:"ul"},"Optimus CLI deploy command")),(0,r.kt)("p",null,"When using Optimus CLI to deploy, either manually or from a CI pipeline, it is advised to use a version control system\nlike git. Here is a simple directory structure that can be used as a template for jobs and datastore resources,\nassuming there are 2 namespaces in a project."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"},".\n\u251c\u2500\u2500 optimus.yaml\n\u251c\u2500\u2500 preset.yaml\n\u251c\u2500\u2500 README.md\n\u251c\u2500\u2500 namespace-1\n\u2502 \u251c\u2500\u2500 jobs\n| \u2502 \u251c\u2500\u2500 job1\n| \u2502 \u251c\u2500\u2500 job2\n| \u2502 \u2514\u2500\u2500 this.yaml\n\u2502 \u2514\u2500\u2500 resources\n| \u251c\u2500\u2500 bigquery\n\u2502 \u2502 \u251c\u2500\u2500 table1\n\u2502 \u2502 \u251c\u2500\u2500 table2\n| | \u2514\u2500\u2500 this.yaml\n\u2502 \u2514\u2500\u2500 postgres\n\u2502 \u2514\u2500\u2500 table1\n\u251c\u2500\u2500 namespace-2\n\u251c\u2500\u2500 jobs\n\u2514\u2500\u2500 resources\n")),(0,r.kt)("p",null,"You might have also noticed there are ",(0,r.kt)("inlineCode",{parentName:"p"},"this.yaml")," files being used in some directories. This file is used to share a\nsingle set of configurations across multiple sub-directories. For example, if you create a file at\n/namespace-1/jobs/this.yaml, then all subdirectories inside /namespaces-1/jobs will inherit this config as defaults.\nIf the same config is specified in subdirectory, then subdirectory will override the parent defaults."),(0,r.kt)("p",null,"For example a this.yaml in ",(0,r.kt)("inlineCode",{parentName:"p"},"/namespace-1/jobs")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nschedule:\n interval: @daily\ntask:\n name: bq2bq\n config:\n BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"\nbehavior:\n depends_on_past: false\n retry:\n count: 1\n delay: 5s\n')),(0,r.kt)("p",null,"and a job.yaml in ",(0,r.kt)("inlineCode",{parentName:"p"},"/namespace-1/jobs/job1")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'name: sample_replace\nowner: optimus@example.io\nschedule:\n start_date: "2020-09-25"\n interval: 0 10 * * *\nbehavior:\n depends_on_past: true\ntask:\n name: bq2bq\n config:\n project: project_name\n dataset: project_dataset\n table: sample_replace\n load_method: REPLACE\n window:\n size: 48h\n offset: 24h\n')),(0,r.kt)("p",null,"will result in final computed job.yaml during deployment as"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample_replace\nowner: optimus@example.io\nschedule:\n start_date: "2020-10-06"\n interval: 0 10 * * *\nbehavior:\n depends_on_past: true\n retry:\n count: 1\n delay: 5s\ntask:\n name: bq2bq\n config:\n project: project_name\n dataset: project_dataset\n table: sample_replace\n load_method: REPLACE\n BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"\n window:\n size: 48h\n offset: 24h\n')),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"Preset (since v0.10.0)")),(0,r.kt)("p",null,"Window preset allows for easier usage of window configuration and can be specified through a YAML file. It is optional in nature and is recommended to be put in the same directory as ",(0,r.kt)("inlineCode",{parentName:"p"},"optimus.yaml")," if being set. For more information on how to utilize window preset, please check ",(0,r.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/intervals-and-windows"},"this page"),"."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6374],{3905:(e,n,t)=>{t.d(n,{Zo:()=>c,kt:()=>f});var a=t(7294);function r(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function i(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);n&&(a=a.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,a)}return t}function o(e){for(var n=1;n=0||(r[t]=e[t]);return r}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var l=a.createContext({}),p=function(e){var n=a.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):o(o({},n),e)),t},c=function(e){var n=p(e.components);return a.createElement(l.Provider,{value:n},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var n=e.children;return a.createElement(a.Fragment,{},n)}},m=a.forwardRef((function(e,n){var t=e.components,r=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(t),m=r,f=u["".concat(l,".").concat(m)]||u[m]||d[m]||i;return t?a.createElement(f,o(o({ref:n},c),{},{components:t})):a.createElement(f,o({ref:n},c))}));function f(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var i=t.length,o=new Array(i);o[0]=m;var s={};for(var l in n)hasOwnProperty.call(n,l)&&(s[l]=n[l]);s.originalType=e,s[u]="string"==typeof e?e:r,o[1]=s;for(var p=2;p{t.r(n),t.d(n,{assets:()=>l,contentTitle:()=>o,default:()=>d,frontMatter:()=>i,metadata:()=>s,toc:()=>p});var a=t(7462),r=(t(7294),t(3905));const i={},o="Organizing Specifications",s={unversionedId:"client-guide/organizing-specifications",id:"client-guide/organizing-specifications",title:"Organizing Specifications",description:"Optimus supports two ways to deploy specifications",source:"@site/docs/client-guide/organizing-specifications.md",sourceDirName:"client-guide",slug:"/client-guide/organizing-specifications",permalink:"/optimus/docs/client-guide/organizing-specifications",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/organizing-specifications.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Uploading Job to Scheduler",permalink:"/optimus/docs/client-guide/uploading-jobs-to-scheduler"},next:{title:"Backup BigQuery Resource",permalink:"/optimus/docs/client-guide/backup-bigquery-resource"}},l={},p=[],c={toc:p},u="wrapper";function d(e){let{components:n,...t}=e;return(0,r.kt)(u,(0,a.Z)({},c,t,{components:n,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"organizing-specifications"},"Organizing Specifications"),(0,r.kt)("p",null,"Optimus supports two ways to deploy specifications"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"REST/GRPC"),(0,r.kt)("li",{parentName:"ul"},"Optimus CLI deploy command")),(0,r.kt)("p",null,"When using Optimus CLI to deploy, either manually or from a CI pipeline, it is advised to use a version control system\nlike git. Here is a simple directory structure that can be used as a template for jobs and datastore resources,\nassuming there are 2 namespaces in a project."),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre"},".\n\u251c\u2500\u2500 optimus.yaml\n\u251c\u2500\u2500 preset.yaml\n\u251c\u2500\u2500 README.md\n\u251c\u2500\u2500 namespace-1\n\u2502 \u251c\u2500\u2500 jobs\n| \u2502 \u251c\u2500\u2500 job1\n| \u2502 \u251c\u2500\u2500 job2\n| \u2502 \u2514\u2500\u2500 this.yaml\n\u2502 \u2514\u2500\u2500 resources\n| \u251c\u2500\u2500 bigquery\n\u2502 \u2502 \u251c\u2500\u2500 table1\n\u2502 \u2502 \u251c\u2500\u2500 table2\n| | \u2514\u2500\u2500 this.yaml\n\u2502 \u2514\u2500\u2500 postgres\n\u2502 \u2514\u2500\u2500 table1\n\u251c\u2500\u2500 namespace-2\n\u251c\u2500\u2500 jobs\n\u2514\u2500\u2500 resources\n")),(0,r.kt)("p",null,"You might have also noticed there are ",(0,r.kt)("inlineCode",{parentName:"p"},"this.yaml")," files being used in some directories. This file is used to share a\nsingle set of configurations across multiple sub-directories. For example, if you create a file at\n/namespace-1/jobs/this.yaml, then all subdirectories inside /namespaces-1/jobs will inherit this config as defaults.\nIf the same config is specified in subdirectory, then subdirectory will override the parent defaults."),(0,r.kt)("p",null,"For example a this.yaml in ",(0,r.kt)("inlineCode",{parentName:"p"},"/namespace-1/jobs")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nschedule:\n interval: @daily\ntask:\n name: bq2bq\n config:\n BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"\nbehavior:\n depends_on_past: false\n retry:\n count: 1\n delay: 5s\n')),(0,r.kt)("p",null,"and a job.yaml in ",(0,r.kt)("inlineCode",{parentName:"p"},"/namespace-1/jobs/job1")),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'name: sample_replace\nowner: optimus@example.io\nschedule:\n start_date: "2020-09-25"\n interval: 0 10 * * *\nbehavior:\n depends_on_past: true\ntask:\n name: bq2bq\n config:\n project: project_name\n dataset: project_dataset\n table: sample_replace\n load_method: REPLACE\n window:\n size: 48h\n offset: 24h\n')),(0,r.kt)("p",null,"will result in final computed job.yaml during deployment as"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-yaml"},'version: 1\nname: sample_replace\nowner: optimus@example.io\nschedule:\n start_date: "2020-10-06"\n interval: 0 10 * * *\nbehavior:\n depends_on_past: true\n retry:\n count: 1\n delay: 5s\ntask:\n name: bq2bq\n config:\n project: project_name\n dataset: project_dataset\n table: sample_replace\n load_method: REPLACE\n BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"\n window:\n size: 48h\n offset: 24h\n')),(0,r.kt)("p",null,(0,r.kt)("strong",{parentName:"p"},"Preset (since v0.10.0)")),(0,r.kt)("p",null,"Window preset allows for easier usage of window configuration and can be specified through a YAML file. It is optional in nature and is recommended to be put in the same directory as ",(0,r.kt)("inlineCode",{parentName:"p"},"optimus.yaml")," if being set. For more information on how to utilize window preset, please check ",(0,r.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/intervals-and-windows"},"this page"),"."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/ae53eb13.a144c5b6.js b/assets/js/ae53eb13.d942e424.js similarity index 98% rename from assets/js/ae53eb13.a144c5b6.js rename to assets/js/ae53eb13.d942e424.js index 17352c7d9a..ff69fbbf2c 100644 --- a/assets/js/ae53eb13.a144c5b6.js +++ b/assets/js/ae53eb13.d942e424.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9439],{3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>f});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function c(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var i=n.createContext({}),s=function(e){var t=n.useContext(i),r=t;return e&&(r="function"==typeof e?e(t):c(c({},t),e)),r},p=function(e){var t=s(e.components);return n.createElement(i.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,i=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=s(r),m=a,f=u["".concat(i,".").concat(m)]||u[m]||d[m]||o;return r?n.createElement(f,c(c({ref:t},p),{},{components:r})):n.createElement(f,c({ref:t},p))}));function f(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,c=new Array(o);c[0]=m;var l={};for(var i in t)hasOwnProperty.call(t,i)&&(l[i]=t[i]);l.originalType=e,l[u]="string"==typeof e?e:a,c[1]=l;for(var s=2;s{r.r(t),r.d(t,{assets:()=>i,contentTitle:()=>c,default:()=>d,frontMatter:()=>o,metadata:()=>l,toc:()=>s});var n=r(7462),a=(r(7294),r(3905));const o={},c="Macros",l={unversionedId:"concepts/macros",id:"concepts/macros",title:"Macros",description:"Macros are special variables that will be replaced by actual values when before execution. Custom macros are not",source:"@site/docs/concepts/macros.md",sourceDirName:"concepts",slug:"/concepts/macros",permalink:"/optimus/docs/concepts/macros",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/macros.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Dependency",permalink:"/optimus/docs/concepts/dependency"},next:{title:"Intervals and Windows",permalink:"/optimus/docs/concepts/intervals-and-windows"}},i={},s=[],p={toc:s},u="wrapper";function d(e){let{components:t,...r}=e;return(0,a.kt)(u,(0,n.Z)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"macros"},"Macros"),(0,a.kt)("p",null,"Macros are special variables that will be replaced by actual values when before execution. Custom macros are not\nsupported yet."),(0,a.kt)("p",null,"Below listed the in built macros supported in optimus."),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Macros"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"{{.DSTART}}"),(0,a.kt)("td",{parentName:"tr",align:null},"start date/datetime of the window as 2021-02-10T10:00:00+00:00 that is, RFC3339")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"{{.DEND}}"),(0,a.kt)("td",{parentName:"tr",align:null},"end date/datetime of the window, as RFC3339")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"{{.JOB_DESTINATION}}"),(0,a.kt)("td",{parentName:"tr",align:null},"full qualified table name used in DML statement")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"{{.EXECUTION_TIME}}"),(0,a.kt)("td",{parentName:"tr",align:null},"timestamp when the specific job run starts")))),(0,a.kt)("p",null,"Take a detailed look at the windows concept and example ",(0,a.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/intervals-and-windows"},"here"),"."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9439],{3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>f});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function c(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var i=n.createContext({}),s=function(e){var t=n.useContext(i),r=t;return e&&(r="function"==typeof e?e(t):c(c({},t),e)),r},p=function(e){var t=s(e.components);return n.createElement(i.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,i=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),u=s(r),m=a,f=u["".concat(i,".").concat(m)]||u[m]||d[m]||o;return r?n.createElement(f,c(c({ref:t},p),{},{components:r})):n.createElement(f,c({ref:t},p))}));function f(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,c=new Array(o);c[0]=m;var l={};for(var i in t)hasOwnProperty.call(t,i)&&(l[i]=t[i]);l.originalType=e,l[u]="string"==typeof e?e:a,c[1]=l;for(var s=2;s{r.r(t),r.d(t,{assets:()=>i,contentTitle:()=>c,default:()=>d,frontMatter:()=>o,metadata:()=>l,toc:()=>s});var n=r(7462),a=(r(7294),r(3905));const o={},c="Macros",l={unversionedId:"concepts/macros",id:"concepts/macros",title:"Macros",description:"Macros are special variables that will be replaced by actual values when before execution. Custom macros are not",source:"@site/docs/concepts/macros.md",sourceDirName:"concepts",slug:"/concepts/macros",permalink:"/optimus/docs/concepts/macros",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/macros.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Dependency",permalink:"/optimus/docs/concepts/dependency"},next:{title:"Intervals and Windows",permalink:"/optimus/docs/concepts/intervals-and-windows"}},i={},s=[],p={toc:s},u="wrapper";function d(e){let{components:t,...r}=e;return(0,a.kt)(u,(0,n.Z)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"macros"},"Macros"),(0,a.kt)("p",null,"Macros are special variables that will be replaced by actual values when before execution. Custom macros are not\nsupported yet."),(0,a.kt)("p",null,"Below listed the in built macros supported in optimus."),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Macros"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"{{.DSTART}}"),(0,a.kt)("td",{parentName:"tr",align:null},"start date/datetime of the window as 2021-02-10T10:00:00+00:00 that is, RFC3339")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"{{.DEND}}"),(0,a.kt)("td",{parentName:"tr",align:null},"end date/datetime of the window, as RFC3339")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"{{.JOB_DESTINATION}}"),(0,a.kt)("td",{parentName:"tr",align:null},"full qualified table name used in DML statement")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"{{.EXECUTION_TIME}}"),(0,a.kt)("td",{parentName:"tr",align:null},"timestamp when the specific job run starts")))),(0,a.kt)("p",null,"Take a detailed look at the windows concept and example ",(0,a.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/intervals-and-windows"},"here"),"."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/b115b19d.6e9b75a8.js b/assets/js/b115b19d.bcda7022.js similarity index 98% rename from assets/js/b115b19d.6e9b75a8.js rename to assets/js/b115b19d.bcda7022.js index 2d1b8b0163..8ee7d1b41f 100644 --- a/assets/js/b115b19d.6e9b75a8.js +++ b/assets/js/b115b19d.bcda7022.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9546],{3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>f});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function s(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var i=n.createContext({}),u=function(e){var t=n.useContext(i),r=t;return e&&(r="function"==typeof e?e(t):s(s({},t),e)),r},p=function(e){var t=u(e.components);return n.createElement(i.Provider,{value:t},e.children)},l="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),l=u(r),m=o,f=l["".concat(i,".").concat(m)]||l[m]||d[m]||a;return r?n.createElement(f,s(s({ref:t},p),{},{components:r})):n.createElement(f,s({ref:t},p))}));function f(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=r.length,s=new Array(a);s[0]=m;var c={};for(var i in t)hasOwnProperty.call(t,i)&&(c[i]=t[i]);c.originalType=e,c[l]="string"==typeof e?e:o,s[1]=c;for(var u=2;u{r.r(t),r.d(t,{assets:()=>i,contentTitle:()=>s,default:()=>d,frontMatter:()=>a,metadata:()=>c,toc:()=>u});var n=r(7462),o=(r(7294),r(3905));const a={},s="Resource",c={unversionedId:"concepts/resource",id:"concepts/resource",title:"Resource",description:"A resource is the representation of the warehouse unit that can be a source or a destination of a transformation job.",source:"@site/docs/concepts/resource.md",sourceDirName:"concepts",slug:"/concepts/resource",permalink:"/optimus/docs/concepts/resource",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/resource.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Namespace",permalink:"/optimus/docs/concepts/namespace"},next:{title:"Job",permalink:"/optimus/docs/concepts/job"}},i={},u=[],p={toc:u},l="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(l,(0,n.Z)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"resource"},"Resource"),(0,o.kt)("p",null,"A resource is the representation of the warehouse unit that can be a source or a destination of a transformation job.\nThe warehouse resources can be created, modified, and be read from Optimus, as well as can be backed up as requested.\nEach warehouse supports a fixed set of resource types and each type has its own specification schema.\nOptimus\u2019 managed warehouse is called Optimus datastore."),(0,o.kt)("p",null,"At the moment, Optimus supports BigQuery datastore for these type of resources:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Dataset"),(0,o.kt)("li",{parentName:"ul"},"Table"),(0,o.kt)("li",{parentName:"ul"},"Standard View"),(0,o.kt)("li",{parentName:"ul"},"External Table")),(0,o.kt)("p",null,(0,o.kt)("em",{parentName:"p"},"Note: BigQuery resource deletion is currently not supported.")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9546],{3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>f});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function s(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var i=n.createContext({}),u=function(e){var t=n.useContext(i),r=t;return e&&(r="function"==typeof e?e(t):s(s({},t),e)),r},p=function(e){var t=u(e.components);return n.createElement(i.Provider,{value:t},e.children)},l="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,a=e.originalType,i=e.parentName,p=c(e,["components","mdxType","originalType","parentName"]),l=u(r),m=o,f=l["".concat(i,".").concat(m)]||l[m]||d[m]||a;return r?n.createElement(f,s(s({ref:t},p),{},{components:r})):n.createElement(f,s({ref:t},p))}));function f(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=r.length,s=new Array(a);s[0]=m;var c={};for(var i in t)hasOwnProperty.call(t,i)&&(c[i]=t[i]);c.originalType=e,c[l]="string"==typeof e?e:o,s[1]=c;for(var u=2;u{r.r(t),r.d(t,{assets:()=>i,contentTitle:()=>s,default:()=>d,frontMatter:()=>a,metadata:()=>c,toc:()=>u});var n=r(7462),o=(r(7294),r(3905));const a={},s="Resource",c={unversionedId:"concepts/resource",id:"concepts/resource",title:"Resource",description:"A resource is the representation of the warehouse unit that can be a source or a destination of a transformation job.",source:"@site/docs/concepts/resource.md",sourceDirName:"concepts",slug:"/concepts/resource",permalink:"/optimus/docs/concepts/resource",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/resource.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Namespace",permalink:"/optimus/docs/concepts/namespace"},next:{title:"Job",permalink:"/optimus/docs/concepts/job"}},i={},u=[],p={toc:u},l="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(l,(0,n.Z)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"resource"},"Resource"),(0,o.kt)("p",null,"A resource is the representation of the warehouse unit that can be a source or a destination of a transformation job.\nThe warehouse resources can be created, modified, and be read from Optimus, as well as can be backed up as requested.\nEach warehouse supports a fixed set of resource types and each type has its own specification schema.\nOptimus\u2019 managed warehouse is called Optimus datastore."),(0,o.kt)("p",null,"At the moment, Optimus supports BigQuery datastore for these type of resources:"),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Dataset"),(0,o.kt)("li",{parentName:"ul"},"Table"),(0,o.kt)("li",{parentName:"ul"},"Standard View"),(0,o.kt)("li",{parentName:"ul"},"External Table")),(0,o.kt)("p",null,(0,o.kt)("em",{parentName:"p"},"Note: BigQuery resource deletion is currently not supported.")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/b14ed82f.a4d8cb0e.js b/assets/js/b14ed82f.4df5e09d.js similarity index 98% rename from assets/js/b14ed82f.a4d8cb0e.js rename to assets/js/b14ed82f.4df5e09d.js index ecb8569c92..e90c01e7e1 100644 --- a/assets/js/b14ed82f.a4d8cb0e.js +++ b/assets/js/b14ed82f.4df5e09d.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[528],{3905:(e,t,a)=>{a.d(t,{Zo:()=>u,kt:()=>y});var r=a(7294);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function o(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var i=r.createContext({}),p=function(e){var t=r.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},u=function(e){var t=p(e.components);return r.createElement(i.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,l=e.originalType,i=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),c=p(a),m=n,y=c["".concat(i,".").concat(m)]||c[m]||d[m]||l;return a?r.createElement(y,o(o({ref:t},u),{},{components:a})):r.createElement(y,o({ref:t},u))}));function y(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=a.length,o=new Array(l);o[0]=m;var s={};for(var i in t)hasOwnProperty.call(t,i)&&(s[i]=t[i]);s.originalType=e,s[c]="string"==typeof e?e:n,o[1]=s;for(var p=2;p{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>o,default:()=>d,frontMatter:()=>l,metadata:()=>s,toc:()=>p});var r=a(7462),n=(a(7294),a(3905));const l={},o="Replay a Job (Backfill)",s={unversionedId:"client-guide/replay-a-job",id:"client-guide/replay-a-job",title:"Replay a Job (Backfill)",description:"Some old dates of a job might need to be re-run (backfill) due to business requirement changes, corrupt data, or other",source:"@site/docs/client-guide/replay-a-job.md",sourceDirName:"client-guide",slug:"/client-guide/replay-a-job",permalink:"/optimus/docs/client-guide/replay-a-job",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/replay-a-job.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Backup BigQuery Resource",permalink:"/optimus/docs/client-guide/backup-bigquery-resource"},next:{title:"Work with Extension",permalink:"/optimus/docs/client-guide/work-with-extension"}},i={},p=[{value:"Run a replay",id:"run-a-replay",level:2},{value:"Get a replay status",id:"get-a-replay-status",level:2},{value:"Get list of replays",id:"get-list-of-replays",level:2}],u={toc:p},c="wrapper";function d(e){let{components:t,...a}=e;return(0,n.kt)(c,(0,r.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,n.kt)("h1",{id:"replay-a-job-backfill"},"Replay a Job (Backfill)"),(0,n.kt)("p",null,"Some old dates of a job might need to be re-run (backfill) due to business requirement changes, corrupt data, or other\nvarious reasons. Optimus provides a way to do this using Replay. Please go through ",(0,n.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/replay-and-backup"},"concepts")," to know more about it."),(0,n.kt)("h2",{id:"run-a-replay"},"Run a replay"),(0,n.kt)("p",null,"To run a replay, run the following command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus replay create {job_name} {start_time} {end_time} [flags]\n")),(0,n.kt)("p",null,"Example:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus replay create sample-job 2023-03-01T00:00:00Z 2023-03-02T15:00:00Z --parallel --project sample-project --namespace-name sample-namespace\n")),(0,n.kt)("p",null,"Replay accepts three arguments, first is the DAG name that is used in Optimus specification, second is the scheduled\nstart time of replay, and third is the scheduled end time (optional) of replay."),(0,n.kt)("p",null,"Once your request has been successfully replayed, this means that Replay has cleared the requested runs in the scheduler.\nPlease wait until the scheduler finishes scheduling and running those tasks."),(0,n.kt)("h2",{id:"get-a-replay-status"},"Get a replay status"),(0,n.kt)("p",null,"You can check the replay status using the replay ID given previously and use in this command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus replay status {replay_id} [flag]\n")),(0,n.kt)("p",null,"You will see the latest replay status including the status of each run of your replay."),(0,n.kt)("h2",{id:"get-list-of-replays"},"Get list of replays"),(0,n.kt)("p",null,"List of recent replay of a project can be checked using this sub command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus replay list [flag]\n")),(0,n.kt)("p",null,"Recent replay ID including the job, time window, replay time, and status will be shown. To check the detailed status\nof a replay, please use the status sub command."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[528],{3905:(e,t,a)=>{a.d(t,{Zo:()=>u,kt:()=>y});var r=a(7294);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function o(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var i=r.createContext({}),p=function(e){var t=r.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},u=function(e){var t=p(e.components);return r.createElement(i.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,l=e.originalType,i=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),c=p(a),m=n,y=c["".concat(i,".").concat(m)]||c[m]||d[m]||l;return a?r.createElement(y,o(o({ref:t},u),{},{components:a})):r.createElement(y,o({ref:t},u))}));function y(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=a.length,o=new Array(l);o[0]=m;var s={};for(var i in t)hasOwnProperty.call(t,i)&&(s[i]=t[i]);s.originalType=e,s[c]="string"==typeof e?e:n,o[1]=s;for(var p=2;p{a.r(t),a.d(t,{assets:()=>i,contentTitle:()=>o,default:()=>d,frontMatter:()=>l,metadata:()=>s,toc:()=>p});var r=a(7462),n=(a(7294),a(3905));const l={},o="Replay a Job (Backfill)",s={unversionedId:"client-guide/replay-a-job",id:"client-guide/replay-a-job",title:"Replay a Job (Backfill)",description:"Some old dates of a job might need to be re-run (backfill) due to business requirement changes, corrupt data, or other",source:"@site/docs/client-guide/replay-a-job.md",sourceDirName:"client-guide",slug:"/client-guide/replay-a-job",permalink:"/optimus/docs/client-guide/replay-a-job",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/replay-a-job.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Backup BigQuery Resource",permalink:"/optimus/docs/client-guide/backup-bigquery-resource"},next:{title:"Work with Extension",permalink:"/optimus/docs/client-guide/work-with-extension"}},i={},p=[{value:"Run a replay",id:"run-a-replay",level:2},{value:"Get a replay status",id:"get-a-replay-status",level:2},{value:"Get list of replays",id:"get-list-of-replays",level:2}],u={toc:p},c="wrapper";function d(e){let{components:t,...a}=e;return(0,n.kt)(c,(0,r.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,n.kt)("h1",{id:"replay-a-job-backfill"},"Replay a Job (Backfill)"),(0,n.kt)("p",null,"Some old dates of a job might need to be re-run (backfill) due to business requirement changes, corrupt data, or other\nvarious reasons. Optimus provides a way to do this using Replay. Please go through ",(0,n.kt)("a",{parentName:"p",href:"/optimus/docs/concepts/replay-and-backup"},"concepts")," to know more about it."),(0,n.kt)("h2",{id:"run-a-replay"},"Run a replay"),(0,n.kt)("p",null,"To run a replay, run the following command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus replay create {job_name} {start_time} {end_time} [flags]\n")),(0,n.kt)("p",null,"Example:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus replay create sample-job 2023-03-01T00:00:00Z 2023-03-02T15:00:00Z --parallel --project sample-project --namespace-name sample-namespace\n")),(0,n.kt)("p",null,"Replay accepts three arguments, first is the DAG name that is used in Optimus specification, second is the scheduled\nstart time of replay, and third is the scheduled end time (optional) of replay."),(0,n.kt)("p",null,"Once your request has been successfully replayed, this means that Replay has cleared the requested runs in the scheduler.\nPlease wait until the scheduler finishes scheduling and running those tasks."),(0,n.kt)("h2",{id:"get-a-replay-status"},"Get a replay status"),(0,n.kt)("p",null,"You can check the replay status using the replay ID given previously and use in this command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus replay status {replay_id} [flag]\n")),(0,n.kt)("p",null,"You will see the latest replay status including the status of each run of your replay."),(0,n.kt)("h2",{id:"get-list-of-replays"},"Get list of replays"),(0,n.kt)("p",null,"List of recent replay of a project can be checked using this sub command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus replay list [flag]\n")),(0,n.kt)("p",null,"Recent replay ID including the job, time window, replay time, and status will be shown. To check the detailed status\nof a replay, please use the status sub command."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/b35a8f38.0bc983f1.js b/assets/js/b35a8f38.80c646f0.js similarity index 98% rename from assets/js/b35a8f38.0bc983f1.js rename to assets/js/b35a8f38.80c646f0.js index 05bb35f954..edbbc61ff5 100644 --- a/assets/js/b35a8f38.0bc983f1.js +++ b/assets/js/b35a8f38.80c646f0.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[648],{3905:(e,t,r)=>{r.d(t,{Zo:()=>c,kt:()=>g});var o=r(7294);function n(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function a(e){for(var t=1;t=0||(n[r]=e[r]);return n}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(n[r]=e[r])}return n}var l=o.createContext({}),p=function(e){var t=o.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):a(a({},t),e)),r},c=function(e){var t=p(e.components);return o.createElement(l.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},d=o.forwardRef((function(e,t){var r=e.components,n=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(r),d=n,g=u["".concat(l,".").concat(d)]||u[d]||m[d]||i;return r?o.createElement(g,a(a({ref:t},c),{},{components:r})):o.createElement(g,a({ref:t},c))}));function g(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=r.length,a=new Array(i);a[0]=d;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[u]="string"==typeof e?e:n,a[1]=s;for(var p=2;p{r.r(t),r.d(t,{assets:()=>l,contentTitle:()=>a,default:()=>m,frontMatter:()=>i,metadata:()=>s,toc:()=>p});var o=r(7462),n=(r(7294),r(3905));const i={},a="DB Migrations",s={unversionedId:"server-guide/db-migrations",id:"server-guide/db-migrations",title:"DB Migrations",description:"Migrate to Specific Version",source:"@site/docs/server-guide/db-migrations.md",sourceDirName:"server-guide",slug:"/server-guide/db-migrations",permalink:"/optimus/docs/server-guide/db-migrations",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/server-guide/db-migrations.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Starting Optimus Server",permalink:"/optimus/docs/server-guide/starting-optimus-server"},next:{title:"Configuration",permalink:"/optimus/docs/client-guide/configuration"}},l={},p=[{value:"Migrate to Specific Version",id:"migrate-to-specific-version",level:2},{value:"Rollback",id:"rollback",level:2},{value:"Export & Upload",id:"export--upload",level:2}],c={toc:p},u="wrapper";function m(e){let{components:t,...r}=e;return(0,n.kt)(u,(0,o.Z)({},c,r,{components:t,mdxType:"MDXLayout"}),(0,n.kt)("h1",{id:"db-migrations"},"DB Migrations"),(0,n.kt)("h2",{id:"migrate-to-specific-version"},"Migrate to Specific Version"),(0,n.kt)("p",null,"Upgrade the DB to the specified migration version."),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus migration to --version [version]\n")),(0,n.kt)("p",null,(0,n.kt)("em",{parentName:"p"},"Note: To migrate to the latest one, running the Optimus\u2019 serve command should be enough. It will migrate to the latest\nautomatically.")),(0,n.kt)("h2",{id:"rollback"},"Rollback"),(0,n.kt)("p",null,"Revert the current active migration to several previous migration versions."),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus migration rollback --count [n]\n")),(0,n.kt)("p",null,"[n]"," is the number of migrations to rollback."),(0,n.kt)("h2",{id:"export--upload"},"Export & Upload"),(0,n.kt)("p",null,"In some cases, due to differences in how Optimus is storing the job and resource specifications in DB in 1 version to\nanother version, you might want to export the jobs in the server to your local YAML files and redeploy it."),(0,n.kt)("p",null,"The export is possible using the following command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job export \u2013-dir job_result\n")),(0,n.kt)("p",null,"This means, you will export all of the jobs from all of the projects in your Optimus server to the job_result directory. It is also possible to run this export command against a single project/namespace/job."),(0,n.kt)("p",null,"For exporting resources, use the following command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource export -dir resource_result\n")),(0,n.kt)("p",null,"Similar to the job command, it is also possible to export resources only for a single project/namespace/resource."))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[648],{3905:(e,t,r)=>{r.d(t,{Zo:()=>c,kt:()=>g});var o=r(7294);function n(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function a(e){for(var t=1;t=0||(n[r]=e[r]);return n}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(n[r]=e[r])}return n}var l=o.createContext({}),p=function(e){var t=o.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):a(a({},t),e)),r},c=function(e){var t=p(e.components);return o.createElement(l.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},d=o.forwardRef((function(e,t){var r=e.components,n=e.mdxType,i=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=p(r),d=n,g=u["".concat(l,".").concat(d)]||u[d]||m[d]||i;return r?o.createElement(g,a(a({ref:t},c),{},{components:r})):o.createElement(g,a({ref:t},c))}));function g(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=r.length,a=new Array(i);a[0]=d;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[u]="string"==typeof e?e:n,a[1]=s;for(var p=2;p{r.r(t),r.d(t,{assets:()=>l,contentTitle:()=>a,default:()=>m,frontMatter:()=>i,metadata:()=>s,toc:()=>p});var o=r(7462),n=(r(7294),r(3905));const i={},a="DB Migrations",s={unversionedId:"server-guide/db-migrations",id:"server-guide/db-migrations",title:"DB Migrations",description:"Migrate to Specific Version",source:"@site/docs/server-guide/db-migrations.md",sourceDirName:"server-guide",slug:"/server-guide/db-migrations",permalink:"/optimus/docs/server-guide/db-migrations",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/server-guide/db-migrations.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Starting Optimus Server",permalink:"/optimus/docs/server-guide/starting-optimus-server"},next:{title:"Configuration",permalink:"/optimus/docs/client-guide/configuration"}},l={},p=[{value:"Migrate to Specific Version",id:"migrate-to-specific-version",level:2},{value:"Rollback",id:"rollback",level:2},{value:"Export & Upload",id:"export--upload",level:2}],c={toc:p},u="wrapper";function m(e){let{components:t,...r}=e;return(0,n.kt)(u,(0,o.Z)({},c,r,{components:t,mdxType:"MDXLayout"}),(0,n.kt)("h1",{id:"db-migrations"},"DB Migrations"),(0,n.kt)("h2",{id:"migrate-to-specific-version"},"Migrate to Specific Version"),(0,n.kt)("p",null,"Upgrade the DB to the specified migration version."),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus migration to --version [version]\n")),(0,n.kt)("p",null,(0,n.kt)("em",{parentName:"p"},"Note: To migrate to the latest one, running the Optimus\u2019 serve command should be enough. It will migrate to the latest\nautomatically.")),(0,n.kt)("h2",{id:"rollback"},"Rollback"),(0,n.kt)("p",null,"Revert the current active migration to several previous migration versions."),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus migration rollback --count [n]\n")),(0,n.kt)("p",null,"[n]"," is the number of migrations to rollback."),(0,n.kt)("h2",{id:"export--upload"},"Export & Upload"),(0,n.kt)("p",null,"In some cases, due to differences in how Optimus is storing the job and resource specifications in DB in 1 version to\nanother version, you might want to export the jobs in the server to your local YAML files and redeploy it."),(0,n.kt)("p",null,"The export is possible using the following command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job export \u2013-dir job_result\n")),(0,n.kt)("p",null,"This means, you will export all of the jobs from all of the projects in your Optimus server to the job_result directory. It is also possible to run this export command against a single project/namespace/job."),(0,n.kt)("p",null,"For exporting resources, use the following command:"),(0,n.kt)("pre",null,(0,n.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus resource export -dir resource_result\n")),(0,n.kt)("p",null,"Similar to the job command, it is also possible to export resources only for a single project/namespace/resource."))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/b7fe478b.8d28011b.js b/assets/js/b7fe478b.3beababb.js similarity index 98% rename from assets/js/b7fe478b.8d28011b.js rename to assets/js/b7fe478b.3beababb.js index 9f448dcb30..d4f1b564f6 100644 --- a/assets/js/b7fe478b.8d28011b.js +++ b/assets/js/b7fe478b.3beababb.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8908],{3905:(e,t,n)=>{n.d(t,{Zo:()=>l,kt:()=>d});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=r.createContext({}),s=function(e){var t=r.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},l=function(e){var t=s(e.components);return r.createElement(p.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},g=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,p=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),u=s(n),g=a,d=u["".concat(p,".").concat(g)]||u[g]||m[g]||o;return n?r.createElement(d,i(i({ref:t},l),{},{components:n})):r.createElement(d,i({ref:t},l))}));function d(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,i=new Array(o);i[0]=g;var c={};for(var p in t)hasOwnProperty.call(t,p)&&(c[p]=t[p]);c.originalType=e,c[u]="string"==typeof e?e:a,i[1]=c;for(var s=2;s{n.r(t),n.d(t,{assets:()=>p,contentTitle:()=>i,default:()=>m,frontMatter:()=>o,metadata:()=>c,toc:()=>s});var r=n(7462),a=(n(7294),n(3905));const o={},i="Managing Project & Namespace",c={unversionedId:"client-guide/managing-project-namespace",id:"client-guide/managing-project-namespace",title:"Managing Project & Namespace",description:"Optimus provides a command to register a new project specified in the client configuration or update if it exists:",source:"@site/docs/client-guide/managing-project-namespace.md",sourceDirName:"client-guide",slug:"/client-guide/managing-project-namespace",permalink:"/optimus/docs/client-guide/managing-project-namespace",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/managing-project-namespace.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Configuration",permalink:"/optimus/docs/client-guide/configuration"},next:{title:"Managing Secrets",permalink:"/optimus/docs/client-guide/managing-secrets"}},p={},s=[],l={toc:s},u="wrapper";function m(e){let{components:t,...n}=e;return(0,a.kt)(u,(0,r.Z)({},l,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"managing-project--namespace"},"Managing Project & Namespace"),(0,a.kt)("p",null,"Optimus provides a command to register a new project specified in the client configuration or update if it exists:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project register\n")),(0,a.kt)("p",null,"You are also allowed to register the namespace by using the with-namespaces flag:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project register --with-namespaces\n")),(0,a.kt)("p",null,"You can also check the project configuration that has been registered in your server using:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project describe\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8908],{3905:(e,t,n)=>{n.d(t,{Zo:()=>l,kt:()=>d});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=r.createContext({}),s=function(e){var t=r.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},l=function(e){var t=s(e.components);return r.createElement(p.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},g=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,p=e.parentName,l=c(e,["components","mdxType","originalType","parentName"]),u=s(n),g=a,d=u["".concat(p,".").concat(g)]||u[g]||m[g]||o;return n?r.createElement(d,i(i({ref:t},l),{},{components:n})):r.createElement(d,i({ref:t},l))}));function d(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,i=new Array(o);i[0]=g;var c={};for(var p in t)hasOwnProperty.call(t,p)&&(c[p]=t[p]);c.originalType=e,c[u]="string"==typeof e?e:a,i[1]=c;for(var s=2;s{n.r(t),n.d(t,{assets:()=>p,contentTitle:()=>i,default:()=>m,frontMatter:()=>o,metadata:()=>c,toc:()=>s});var r=n(7462),a=(n(7294),n(3905));const o={},i="Managing Project & Namespace",c={unversionedId:"client-guide/managing-project-namespace",id:"client-guide/managing-project-namespace",title:"Managing Project & Namespace",description:"Optimus provides a command to register a new project specified in the client configuration or update if it exists:",source:"@site/docs/client-guide/managing-project-namespace.md",sourceDirName:"client-guide",slug:"/client-guide/managing-project-namespace",permalink:"/optimus/docs/client-guide/managing-project-namespace",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/managing-project-namespace.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Configuration",permalink:"/optimus/docs/client-guide/configuration"},next:{title:"Managing Secrets",permalink:"/optimus/docs/client-guide/managing-secrets"}},p={},s=[],l={toc:s},u="wrapper";function m(e){let{components:t,...n}=e;return(0,a.kt)(u,(0,r.Z)({},l,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"managing-project--namespace"},"Managing Project & Namespace"),(0,a.kt)("p",null,"Optimus provides a command to register a new project specified in the client configuration or update if it exists:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project register\n")),(0,a.kt)("p",null,"You are also allowed to register the namespace by using the with-namespaces flag:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project register --with-namespaces\n")),(0,a.kt)("p",null,"You can also check the project configuration that has been registered in your server using:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus project describe\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/b95ea484.07276431.js b/assets/js/b95ea484.bfba3a89.js similarity index 98% rename from assets/js/b95ea484.07276431.js rename to assets/js/b95ea484.bfba3a89.js index ec743487ad..99a1b3dcb8 100644 --- a/assets/js/b95ea484.07276431.js +++ b/assets/js/b95ea484.bfba3a89.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3488],{3905:(e,t,r)=>{r.d(t,{Zo:()=>u,kt:()=>d});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var c=n.createContext({}),s=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},u=function(e){var t=s(e.components);return n.createElement(c.Provider,{value:t},e.children)},l="mdxType",f={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,a=e.originalType,c=e.parentName,u=p(e,["components","mdxType","originalType","parentName"]),l=s(r),m=o,d=l["".concat(c,".").concat(m)]||l[m]||f[m]||a;return r?n.createElement(d,i(i({ref:t},u),{},{components:r})):n.createElement(d,i({ref:t},u))}));function d(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=r.length,i=new Array(a);i[0]=m;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p[l]="string"==typeof e?e:o,i[1]=p;for(var s=2;s{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>f,frontMatter:()=>a,metadata:()=>p,toc:()=>s});var n=r(7462),o=(r(7294),r(3905));const a={},i="API",p={unversionedId:"reference/api",id:"reference/api",title:"API",description:"Optimus service supports REST and GRPC for interaction, currently on the same port.",source:"@site/docs/reference/api.md",sourceDirName:"reference",slug:"/reference/api",permalink:"/optimus/docs/reference/api",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/reference/api.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Developer Environment Setup",permalink:"/optimus/docs/contribute/developer-env-setup"},next:{title:"Metrics",permalink:"/optimus/docs/reference/metrics"}},c={},s=[],u={toc:s},l="wrapper";function f(e){let{components:t,...r}=e;return(0,o.kt)(l,(0,n.Z)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"api"},"API"),(0,o.kt)("p",null,"Optimus service supports REST and GRPC for interaction, currently on the same port."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("a",{parentName:"li",href:"https://github.com/goto/optimus/blob/a32e35aef61e5d51672b1afc131e9ea828cff1a5/api/third_party/openapi/goto/optimus/core/v1beta1/runtime.swagger.json"},"REST API")),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("a",{parentName:"li",href:"https://github.com/goto/proton/blob/ef83b9e9248e064a1c366da4fe07b3068266fe59/goto/optimus/core/v1beta1/runtime.proto"},"GRPC"))))}f.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3488],{3905:(e,t,r)=>{r.d(t,{Zo:()=>u,kt:()=>d});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var c=n.createContext({}),s=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},u=function(e){var t=s(e.components);return n.createElement(c.Provider,{value:t},e.children)},l="mdxType",f={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,a=e.originalType,c=e.parentName,u=p(e,["components","mdxType","originalType","parentName"]),l=s(r),m=o,d=l["".concat(c,".").concat(m)]||l[m]||f[m]||a;return r?n.createElement(d,i(i({ref:t},u),{},{components:r})):n.createElement(d,i({ref:t},u))}));function d(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=r.length,i=new Array(a);i[0]=m;var p={};for(var c in t)hasOwnProperty.call(t,c)&&(p[c]=t[c]);p.originalType=e,p[l]="string"==typeof e?e:o,i[1]=p;for(var s=2;s{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>f,frontMatter:()=>a,metadata:()=>p,toc:()=>s});var n=r(7462),o=(r(7294),r(3905));const a={},i="API",p={unversionedId:"reference/api",id:"reference/api",title:"API",description:"Optimus service supports REST and GRPC for interaction, currently on the same port.",source:"@site/docs/reference/api.md",sourceDirName:"reference",slug:"/reference/api",permalink:"/optimus/docs/reference/api",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/reference/api.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Developer Environment Setup",permalink:"/optimus/docs/contribute/developer-env-setup"},next:{title:"Metrics",permalink:"/optimus/docs/reference/metrics"}},c={},s=[],u={toc:s},l="wrapper";function f(e){let{components:t,...r}=e;return(0,o.kt)(l,(0,n.Z)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"api"},"API"),(0,o.kt)("p",null,"Optimus service supports REST and GRPC for interaction, currently on the same port."),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("a",{parentName:"li",href:"https://github.com/goto/optimus/blob/a32e35aef61e5d51672b1afc131e9ea828cff1a5/api/third_party/openapi/goto/optimus/core/v1beta1/runtime.swagger.json"},"REST API")),(0,o.kt)("li",{parentName:"ul"},(0,o.kt)("a",{parentName:"li",href:"https://github.com/goto/proton/blob/ef83b9e9248e064a1c366da4fe07b3068266fe59/goto/optimus/core/v1beta1/runtime.proto"},"GRPC"))))}f.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/bf534763.df52fb30.js b/assets/js/bf534763.af003002.js similarity index 99% rename from assets/js/bf534763.df52fb30.js rename to assets/js/bf534763.af003002.js index 4431f0b721..d2b4e7420f 100644 --- a/assets/js/bf534763.df52fb30.js +++ b/assets/js/bf534763.af003002.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3771],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>m});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function a(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=r.createContext({}),c=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},p=function(e){var t=c(e.components);return r.createElement(l.Provider,{value:t},e.children)},d="mdxType",h={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},u=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=o,m=d["".concat(l,".").concat(u)]||d[u]||h[u]||i;return n?r.createElement(m,a(a({ref:t},p),{},{components:n})):r.createElement(m,a({ref:t},p))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=u;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[d]="string"==typeof e?e:o,a[1]=s;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>a,default:()=>h,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var r=n(7462),o=(n(7294),n(3905));const i={},a="About This Directory",s={unversionedId:"rfcs/README",id:"rfcs/README",title:"About This Directory",description:"This directory contains RFCs (design documents) that describe proposed major changes to Optimus.",source:"@site/docs/rfcs/README.md",sourceDirName:"rfcs",slug:"/rfcs/",permalink:"/optimus/docs/rfcs/",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/README.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{}},l={},c=[],p={toc:c},d="wrapper";function h(e){let{components:t,...n}=e;return(0,o.kt)(d,(0,r.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"about-this-directory"},"About This Directory"),(0,o.kt)("p",null,"This directory contains RFCs (design documents) that describe proposed major changes to Optimus."),(0,o.kt)("h1",{id:"the-why-of-rfcs"},"The Why of RFCs"),(0,o.kt)("p",null,"An RFC provides a high-level description of a major change or enhancement to Optimus. The high-level description allows a reviewer to critique and poke holes in a design without getting lost in the particulars of code."),(0,o.kt)("p",null,"An RFC is a form of communication aimed at both spreading and gathering knowledge, though it is not the sole means of accomplishing either task. Prototypes, tech notes, github issues, comments in code, commit messages and in-person discussions are valid alternatives depending on the situation."),(0,o.kt)("p",null,"At its best, an RFC clearly and concisely describes the high-level architecture of a project giving confidence to all involved. At its worst, an RFC focuses on unimportant details, fosters discussion that stymies progress, or demoralizes the author with the complexity of their undertaking."),(0,o.kt)("h1",{id:"the-when-and-how-of-rfcs"},"The When and How of RFCs"),(0,o.kt)("p",null,"When to write an RFC is nuanced and there are no firm rules. General guidance is to write an RFC before embarking on a significant or complex project that will be spread over multiple pull requests (PRs), and when multiple alternatives need to be considered and there is no obvious best approach. A project involving multiple people is a good signal an RFC is warranted. (Similarly, a project worthy of an RFC often requires multiple engineers worth of effort). Note that this guidance is intentionally vague. Many complex projects spread over multiple PRs do not require an RFC, though they do require adequate communication and discussion."),(0,o.kt)("p",null,"It is encouraged to develop a prototype concurrently with writing the RFC. One of the significant benefits of an RFC is that it forces bigger picture thinking which reviewers can then disect. In contrast, a prototype forces the details to be considered, shedding light on the unknown and helping to ensure that the RFC focuses on the important design considerations."),(0,o.kt)("p",null,"An RFC should be a high-level description which does not require formal correctness. There is utility in conciseness. Do not overspecify the details in the RFC as doing so can bury the reviewer in minutiae. If you've never written an RFC before consider partnering with a more experienced engineer for guidance and to help shepherd your RFC through the process."),(0,o.kt)("h1",{id:"rfc-process"},"RFC Process"),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Every RFC should have a dedicated reviewer familiar with the RFC's subject area.")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Copy ",(0,o.kt)("inlineCode",{parentName:"p"},"00000000_template.md")," to a new file and fill in the details. Commit this version in your own fork of the repository or a branch. Your commit message (and corresponding pull request) should include the prefix ",(0,o.kt)("inlineCode",{parentName:"p"},"rfc"),". Eg: ",(0,o.kt)("inlineCode",{parentName:"p"},"rfc: edit RFC template")),(0,o.kt)("p",{parentName:"li"},"If you are a creative person, you may prefer to start with this blank slate, write your prose and then later check that all necessary topics have been covered."),(0,o.kt)("p",{parentName:"li"},"If you feel intimidated by a blank template, you can instead peruse the list of requested topics and use the questions in there as writing prompt.")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Submit a pull request (PR) to add your new file to the main repository. Each RFC should get its own pull request; do not combine RFCs with other files."),(0,o.kt)("p",{parentName:"li"},'Note: you can send a PR before the RFC is complete in order to solicit input about what to write in the RFC. In this case, include the term "',"[WIP]",'" (work in progress) in the PR title & mark the PR as draft until you are confident the RFC is complete and can be reviewed.')),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Go through the PR review, iterating on the RFC to answer questions and concerns from the reviewer(s). The duration of this process should be related to the complexity of the project. If you or the reviewers seem to be at an impasse, consider in-person discussions or a prototype. There is no minimum time required to leave an RFC open for review. There is also no prohibition about halting or reversing work on an accepted RFC if a problem is discovered during implementation."),(0,o.kt)("p",{parentName:"li"},"Reviewers should be conscious of their own limitations and ask for other engineers to look at specific areas if necessary.")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Once discussion has settled and the RFC has received an LGTM from the reviewer(s):"),(0,o.kt)("ul",{parentName:"li"},(0,o.kt)("li",{parentName:"ul"},"change the ",(0,o.kt)("inlineCode",{parentName:"li"},"Status")," field of the document to ",(0,o.kt)("inlineCode",{parentName:"li"},"in-progress"),";"),(0,o.kt)("li",{parentName:"ul"},"rename the RFC document to prefix it with the current date (",(0,o.kt)("inlineCode",{parentName:"li"},"YYYYMMDD_"),");"),(0,o.kt)("li",{parentName:"ul"},"update the ",(0,o.kt)("inlineCode",{parentName:"li"},"RFC PR")," field;"),(0,o.kt)("li",{parentName:"ul"},"and merge the PR."))),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Once the changes in the RFC have been implemented and merged, change the ",(0,o.kt)("inlineCode",{parentName:"p"},"Status")," field of the document from ",(0,o.kt)("inlineCode",{parentName:"p"},"in-progress")," to ",(0,o.kt)("inlineCode",{parentName:"p"},"completed"),". If subsequent developments render an RFC obsolete, change its status to ",(0,o.kt)("inlineCode",{parentName:"p"},"obsolete"),". When you mark a RFC as obsolete, ensure that its text references the other RFCs or PRs that make it obsolete."))))}h.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3771],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>m});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function a(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=r.createContext({}),c=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},p=function(e){var t=c(e.components);return r.createElement(l.Provider,{value:t},e.children)},d="mdxType",h={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},u=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(n),u=o,m=d["".concat(l,".").concat(u)]||d[u]||h[u]||i;return n?r.createElement(m,a(a({ref:t},p),{},{components:n})):r.createElement(m,a({ref:t},p))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,a=new Array(i);a[0]=u;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[d]="string"==typeof e?e:o,a[1]=s;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>a,default:()=>h,frontMatter:()=>i,metadata:()=>s,toc:()=>c});var r=n(7462),o=(n(7294),n(3905));const i={},a="About This Directory",s={unversionedId:"rfcs/README",id:"rfcs/README",title:"About This Directory",description:"This directory contains RFCs (design documents) that describe proposed major changes to Optimus.",source:"@site/docs/rfcs/README.md",sourceDirName:"rfcs",slug:"/rfcs/",permalink:"/optimus/docs/rfcs/",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/README.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{}},l={},c=[],p={toc:c},d="wrapper";function h(e){let{components:t,...n}=e;return(0,o.kt)(d,(0,r.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"about-this-directory"},"About This Directory"),(0,o.kt)("p",null,"This directory contains RFCs (design documents) that describe proposed major changes to Optimus."),(0,o.kt)("h1",{id:"the-why-of-rfcs"},"The Why of RFCs"),(0,o.kt)("p",null,"An RFC provides a high-level description of a major change or enhancement to Optimus. The high-level description allows a reviewer to critique and poke holes in a design without getting lost in the particulars of code."),(0,o.kt)("p",null,"An RFC is a form of communication aimed at both spreading and gathering knowledge, though it is not the sole means of accomplishing either task. Prototypes, tech notes, github issues, comments in code, commit messages and in-person discussions are valid alternatives depending on the situation."),(0,o.kt)("p",null,"At its best, an RFC clearly and concisely describes the high-level architecture of a project giving confidence to all involved. At its worst, an RFC focuses on unimportant details, fosters discussion that stymies progress, or demoralizes the author with the complexity of their undertaking."),(0,o.kt)("h1",{id:"the-when-and-how-of-rfcs"},"The When and How of RFCs"),(0,o.kt)("p",null,"When to write an RFC is nuanced and there are no firm rules. General guidance is to write an RFC before embarking on a significant or complex project that will be spread over multiple pull requests (PRs), and when multiple alternatives need to be considered and there is no obvious best approach. A project involving multiple people is a good signal an RFC is warranted. (Similarly, a project worthy of an RFC often requires multiple engineers worth of effort). Note that this guidance is intentionally vague. Many complex projects spread over multiple PRs do not require an RFC, though they do require adequate communication and discussion."),(0,o.kt)("p",null,"It is encouraged to develop a prototype concurrently with writing the RFC. One of the significant benefits of an RFC is that it forces bigger picture thinking which reviewers can then disect. In contrast, a prototype forces the details to be considered, shedding light on the unknown and helping to ensure that the RFC focuses on the important design considerations."),(0,o.kt)("p",null,"An RFC should be a high-level description which does not require formal correctness. There is utility in conciseness. Do not overspecify the details in the RFC as doing so can bury the reviewer in minutiae. If you've never written an RFC before consider partnering with a more experienced engineer for guidance and to help shepherd your RFC through the process."),(0,o.kt)("h1",{id:"rfc-process"},"RFC Process"),(0,o.kt)("ol",null,(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Every RFC should have a dedicated reviewer familiar with the RFC's subject area.")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Copy ",(0,o.kt)("inlineCode",{parentName:"p"},"00000000_template.md")," to a new file and fill in the details. Commit this version in your own fork of the repository or a branch. Your commit message (and corresponding pull request) should include the prefix ",(0,o.kt)("inlineCode",{parentName:"p"},"rfc"),". Eg: ",(0,o.kt)("inlineCode",{parentName:"p"},"rfc: edit RFC template")),(0,o.kt)("p",{parentName:"li"},"If you are a creative person, you may prefer to start with this blank slate, write your prose and then later check that all necessary topics have been covered."),(0,o.kt)("p",{parentName:"li"},"If you feel intimidated by a blank template, you can instead peruse the list of requested topics and use the questions in there as writing prompt.")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Submit a pull request (PR) to add your new file to the main repository. Each RFC should get its own pull request; do not combine RFCs with other files."),(0,o.kt)("p",{parentName:"li"},'Note: you can send a PR before the RFC is complete in order to solicit input about what to write in the RFC. In this case, include the term "',"[WIP]",'" (work in progress) in the PR title & mark the PR as draft until you are confident the RFC is complete and can be reviewed.')),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Go through the PR review, iterating on the RFC to answer questions and concerns from the reviewer(s). The duration of this process should be related to the complexity of the project. If you or the reviewers seem to be at an impasse, consider in-person discussions or a prototype. There is no minimum time required to leave an RFC open for review. There is also no prohibition about halting or reversing work on an accepted RFC if a problem is discovered during implementation."),(0,o.kt)("p",{parentName:"li"},"Reviewers should be conscious of their own limitations and ask for other engineers to look at specific areas if necessary.")),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Once discussion has settled and the RFC has received an LGTM from the reviewer(s):"),(0,o.kt)("ul",{parentName:"li"},(0,o.kt)("li",{parentName:"ul"},"change the ",(0,o.kt)("inlineCode",{parentName:"li"},"Status")," field of the document to ",(0,o.kt)("inlineCode",{parentName:"li"},"in-progress"),";"),(0,o.kt)("li",{parentName:"ul"},"rename the RFC document to prefix it with the current date (",(0,o.kt)("inlineCode",{parentName:"li"},"YYYYMMDD_"),");"),(0,o.kt)("li",{parentName:"ul"},"update the ",(0,o.kt)("inlineCode",{parentName:"li"},"RFC PR")," field;"),(0,o.kt)("li",{parentName:"ul"},"and merge the PR."))),(0,o.kt)("li",{parentName:"ol"},(0,o.kt)("p",{parentName:"li"},"Once the changes in the RFC have been implemented and merged, change the ",(0,o.kt)("inlineCode",{parentName:"p"},"Status")," field of the document from ",(0,o.kt)("inlineCode",{parentName:"p"},"in-progress")," to ",(0,o.kt)("inlineCode",{parentName:"p"},"completed"),". If subsequent developments render an RFC obsolete, change its status to ",(0,o.kt)("inlineCode",{parentName:"p"},"obsolete"),". When you mark a RFC as obsolete, ensure that its text references the other RFCs or PRs that make it obsolete."))))}h.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/c20e3951.c0e35d10.js b/assets/js/c20e3951.ace81d80.js similarity index 99% rename from assets/js/c20e3951.c0e35d10.js rename to assets/js/c20e3951.ace81d80.js index 48bb13af9c..413dd9006a 100644 --- a/assets/js/c20e3951.c0e35d10.js +++ b/assets/js/c20e3951.ace81d80.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2314],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>m});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},h=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,s=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),c=p(n),h=a,m=c["".concat(s,".").concat(h)]||c[h]||d[h]||o;return n?r.createElement(m,l(l({ref:t},u),{},{components:n})):r.createElement(m,l({ref:t},u))}));function m(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,l=new Array(o);l[0]=h;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i[c]="string"==typeof e?e:a,l[1]=i;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>d,frontMatter:()=>o,metadata:()=>i,toc:()=>p});var r=n(7462),a=(n(7294),n(3905));const o={},l=void 0,i={unversionedId:"rfcs/replay_rate_limiting",id:"rfcs/replay_rate_limiting",title:"replay_rate_limiting",description:"- Feature Name: Replay Rate Limit",source:"@site/docs/rfcs/20220525_replay_rate_limiting.md",sourceDirName:"rfcs",slug:"/rfcs/replay_rate_limiting",permalink:"/optimus/docs/rfcs/replay_rate_limiting",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220525_replay_rate_limiting.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220525,frontMatter:{}},s={},p=[{value:"Background :",id:"background-",level:2},{value:"Approach :",id:"approach-",level:2},{value:"Reduce the load of Execution Project",id:"reduce-the-load-of-execution-project",level:3},{value:"Reduce the load of Airflow Default Pool",id:"reduce-the-load-of-airflow-default-pool",level:3}],u={toc:p},c="wrapper";function d(e){let{components:t,...n}=e;return(0,a.kt)(c,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"Feature Name: Replay Rate Limit"),(0,a.kt)("li",{parentName:"ul"},"Status: Draft"),(0,a.kt)("li",{parentName:"ul"},"Start Date: 2022-05-25"),(0,a.kt)("li",{parentName:"ul"},"Authors: Arinda")),(0,a.kt)("h1",{id:"summary"},"Summary"),(0,a.kt)("p",null,"Replay is using the same scheduler pool and executor slots as the scheduled jobs. Current Replay is predicted to be\ncausing issues and impacting the scheduled jobs, which might impacting the SLA of users. There should be a mechanism to\nuse a different pool and slots and configurable by the users so any backfilling will not impact and causing delay of\nthe scheduled jobs."),(0,a.kt)("h1",{id:"technical-design"},"Technical Design"),(0,a.kt)("h2",{id:"background-"},"Background :"),(0,a.kt)("p",null,"Currently, for bq2bq task the job is being executed using the project that specified in EXECUTION_PROJECT. This is also\napplied when the jobs are run through replay command. How Replay works is by clearing runs of the requested jobs and its\ndependency (if required), and there is no difference between the process of scheduled/manual job and replay. This also\nmeans the execution project and scheduler pool is also the same."),(0,a.kt)("h2",{id:"approach-"},"Approach :"),(0,a.kt)("h3",{id:"reduce-the-load-of-execution-project"},"Reduce the load of Execution Project"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"There will be ",(0,a.kt)("inlineCode",{parentName:"li"},"REPLAY_EXECUTION_PROJECT")," configuration, it can be set through the task config, project or namespace config."),(0,a.kt)("li",{parentName:"ul"},"When building instances, Optimus will check if there is a replay request for the particular job and date, if yes,\n",(0,a.kt)("inlineCode",{parentName:"li"},"EXECUTION_PROJECT")," will be replaced with ",(0,a.kt)("inlineCode",{parentName:"li"},"REPLAY_EXECUTION_PROJECT"),".")),(0,a.kt)("p",null,"In job specs, task level:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-yaml"},"task:\n name: bq2bq\n config:\n PROJECT: sample-project\n DATASET: sample_dataset\n TABLE: sample_table\n SQL_TYPE: STANDARD\n LOAD_METHOD: REPLACE\n EXECUTION_PROJECT: main-executor-project\n REPLAY_EXECUTION_PROJECT: replay-executor-project\n")),(0,a.kt)("h3",{id:"reduce-the-load-of-airflow-default-pool"},"Reduce the load of Airflow Default Pool"),(0,a.kt)("p",null,"Initially, there are 3 mechanism to be considered for this."),(0,a.kt)("ol",null,(0,a.kt)("li",{parentName:"ol"},"Differentiating Airflow pool for Replay."),(0,a.kt)("li",{parentName:"ol"},"Trigger a new run for Replay (not using current clear method)."),(0,a.kt)("li",{parentName:"ol"},"Limit the slot through Optimus.")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"For the first option"),", Airflow pool is configured through DAG. This approach requires task to be configured using\neach of the expected pool, for example ",(0,a.kt)("inlineCode",{parentName:"p"},"bq2bq-default")," and ",(0,a.kt)("inlineCode",{parentName:"p"},"bq2bq-replay"),". The task to be selected is being decided\nusing ",(0,a.kt)("inlineCode",{parentName:"p"},"BranchPythonOperator"),". This is not preferred as there will numerous task being set and visualize, and might\nconfuse users."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"For the second option"),", the replay run is being triggered using API, not by using the clear run method. However,\nAirflow is not accepting a pool configuration, thus it will still goes to the same pool."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"The third option"),", is limiting the slot through Optimus. There will be configuration in project level to set the\nnumber of slots Replay can occupy."),(0,a.kt)("p",null,"Project configuration"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-yaml"},"project:\n name: sample-project\n config:\n storage_path: gs://sample-bucket\n max_replay_runs_per_project: 15\n max_replay_runs_per_dag: 5\n")),(0,a.kt)("p",null,"Lets say there is only 15 Replay slots at a time, and there are 40 runs requests comes. Replay will clear the first 15,\nput the rest 25 in queue, and Replay will keep checking on some interval, of whether the process has been completed and\ncan take another runs."),(0,a.kt)("p",null,"Picking which runs to be processed:"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"Based on Replay request time (FIFO)"),(0,a.kt)("li",{parentName:"ul"},"Based on the slot per dag. If the overall is set to 15 but the slot per dag is only 5, then a request to replay a job\nthat has 10 runs cannot be done in one go.")),(0,a.kt)("p",null,"There should be a default value for each of the configuration."),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"Max_replay_runs_per_project: default 10"),(0,a.kt)("li",{parentName:"ul"},"Max_replay_runs_per_dag: default 5")),(0,a.kt)("p",null,"Optimus will check on how many Replay job is currently running, and how many task run is running for each job. Additional\nchanges might be needed on Replay side for optimization, to avoid calling Airflow API in every interval (Replay syncer\nresponsibility) and fetch data from Replay table instead."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2314],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>m});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function l(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},h=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,s=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),c=p(n),h=a,m=c["".concat(s,".").concat(h)]||c[h]||d[h]||o;return n?r.createElement(m,l(l({ref:t},u),{},{components:n})):r.createElement(m,l({ref:t},u))}));function m(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,l=new Array(o);l[0]=h;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i[c]="string"==typeof e?e:a,l[1]=i;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>d,frontMatter:()=>o,metadata:()=>i,toc:()=>p});var r=n(7462),a=(n(7294),n(3905));const o={},l=void 0,i={unversionedId:"rfcs/replay_rate_limiting",id:"rfcs/replay_rate_limiting",title:"replay_rate_limiting",description:"- Feature Name: Replay Rate Limit",source:"@site/docs/rfcs/20220525_replay_rate_limiting.md",sourceDirName:"rfcs",slug:"/rfcs/replay_rate_limiting",permalink:"/optimus/docs/rfcs/replay_rate_limiting",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220525_replay_rate_limiting.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220525,frontMatter:{}},s={},p=[{value:"Background :",id:"background-",level:2},{value:"Approach :",id:"approach-",level:2},{value:"Reduce the load of Execution Project",id:"reduce-the-load-of-execution-project",level:3},{value:"Reduce the load of Airflow Default Pool",id:"reduce-the-load-of-airflow-default-pool",level:3}],u={toc:p},c="wrapper";function d(e){let{components:t,...n}=e;return(0,a.kt)(c,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"Feature Name: Replay Rate Limit"),(0,a.kt)("li",{parentName:"ul"},"Status: Draft"),(0,a.kt)("li",{parentName:"ul"},"Start Date: 2022-05-25"),(0,a.kt)("li",{parentName:"ul"},"Authors: Arinda")),(0,a.kt)("h1",{id:"summary"},"Summary"),(0,a.kt)("p",null,"Replay is using the same scheduler pool and executor slots as the scheduled jobs. Current Replay is predicted to be\ncausing issues and impacting the scheduled jobs, which might impacting the SLA of users. There should be a mechanism to\nuse a different pool and slots and configurable by the users so any backfilling will not impact and causing delay of\nthe scheduled jobs."),(0,a.kt)("h1",{id:"technical-design"},"Technical Design"),(0,a.kt)("h2",{id:"background-"},"Background :"),(0,a.kt)("p",null,"Currently, for bq2bq task the job is being executed using the project that specified in EXECUTION_PROJECT. This is also\napplied when the jobs are run through replay command. How Replay works is by clearing runs of the requested jobs and its\ndependency (if required), and there is no difference between the process of scheduled/manual job and replay. This also\nmeans the execution project and scheduler pool is also the same."),(0,a.kt)("h2",{id:"approach-"},"Approach :"),(0,a.kt)("h3",{id:"reduce-the-load-of-execution-project"},"Reduce the load of Execution Project"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"There will be ",(0,a.kt)("inlineCode",{parentName:"li"},"REPLAY_EXECUTION_PROJECT")," configuration, it can be set through the task config, project or namespace config."),(0,a.kt)("li",{parentName:"ul"},"When building instances, Optimus will check if there is a replay request for the particular job and date, if yes,\n",(0,a.kt)("inlineCode",{parentName:"li"},"EXECUTION_PROJECT")," will be replaced with ",(0,a.kt)("inlineCode",{parentName:"li"},"REPLAY_EXECUTION_PROJECT"),".")),(0,a.kt)("p",null,"In job specs, task level:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-yaml"},"task:\n name: bq2bq\n config:\n PROJECT: sample-project\n DATASET: sample_dataset\n TABLE: sample_table\n SQL_TYPE: STANDARD\n LOAD_METHOD: REPLACE\n EXECUTION_PROJECT: main-executor-project\n REPLAY_EXECUTION_PROJECT: replay-executor-project\n")),(0,a.kt)("h3",{id:"reduce-the-load-of-airflow-default-pool"},"Reduce the load of Airflow Default Pool"),(0,a.kt)("p",null,"Initially, there are 3 mechanism to be considered for this."),(0,a.kt)("ol",null,(0,a.kt)("li",{parentName:"ol"},"Differentiating Airflow pool for Replay."),(0,a.kt)("li",{parentName:"ol"},"Trigger a new run for Replay (not using current clear method)."),(0,a.kt)("li",{parentName:"ol"},"Limit the slot through Optimus.")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"For the first option"),", Airflow pool is configured through DAG. This approach requires task to be configured using\neach of the expected pool, for example ",(0,a.kt)("inlineCode",{parentName:"p"},"bq2bq-default")," and ",(0,a.kt)("inlineCode",{parentName:"p"},"bq2bq-replay"),". The task to be selected is being decided\nusing ",(0,a.kt)("inlineCode",{parentName:"p"},"BranchPythonOperator"),". This is not preferred as there will numerous task being set and visualize, and might\nconfuse users."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"For the second option"),", the replay run is being triggered using API, not by using the clear run method. However,\nAirflow is not accepting a pool configuration, thus it will still goes to the same pool."),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"The third option"),", is limiting the slot through Optimus. There will be configuration in project level to set the\nnumber of slots Replay can occupy."),(0,a.kt)("p",null,"Project configuration"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-yaml"},"project:\n name: sample-project\n config:\n storage_path: gs://sample-bucket\n max_replay_runs_per_project: 15\n max_replay_runs_per_dag: 5\n")),(0,a.kt)("p",null,"Lets say there is only 15 Replay slots at a time, and there are 40 runs requests comes. Replay will clear the first 15,\nput the rest 25 in queue, and Replay will keep checking on some interval, of whether the process has been completed and\ncan take another runs."),(0,a.kt)("p",null,"Picking which runs to be processed:"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"Based on Replay request time (FIFO)"),(0,a.kt)("li",{parentName:"ul"},"Based on the slot per dag. If the overall is set to 15 but the slot per dag is only 5, then a request to replay a job\nthat has 10 runs cannot be done in one go.")),(0,a.kt)("p",null,"There should be a default value for each of the configuration."),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"Max_replay_runs_per_project: default 10"),(0,a.kt)("li",{parentName:"ul"},"Max_replay_runs_per_dag: default 5")),(0,a.kt)("p",null,"Optimus will check on how many Replay job is currently running, and how many task run is running for each job. Additional\nchanges might be needed on Replay side for optimization, to avoid calling Airflow API in every interval (Replay syncer\nresponsibility) and fetch data from Replay table instead."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/cf34028a.c343a909.js b/assets/js/cf34028a.4ba3eef0.js similarity index 98% rename from assets/js/cf34028a.c343a909.js rename to assets/js/cf34028a.4ba3eef0.js index 1abcd1232e..10b2bddb42 100644 --- a/assets/js/cf34028a.c343a909.js +++ b/assets/js/cf34028a.4ba3eef0.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2147],{3905:(e,t,r)=>{r.d(t,{Zo:()=>l,kt:()=>f});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function c(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function a(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var c=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var i=n.createContext({}),p=function(e){var t=n.useContext(i),r=t;return e&&(r="function"==typeof e?e(t):a(a({},t),e)),r},l=function(e){var t=p(e.components);return n.createElement(i.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,c=e.originalType,i=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),u=p(r),m=o,f=u["".concat(i,".").concat(m)]||u[m]||d[m]||c;return r?n.createElement(f,a(a({ref:t},l),{},{components:r})):n.createElement(f,a({ref:t},l))}));function f(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var c=r.length,a=new Array(c);a[0]=m;var s={};for(var i in t)hasOwnProperty.call(t,i)&&(s[i]=t[i]);s.originalType=e,s[u]="string"==typeof e?e:o,a[1]=s;for(var p=2;p{r.r(t),r.d(t,{assets:()=>i,contentTitle:()=>a,default:()=>d,frontMatter:()=>c,metadata:()=>s,toc:()=>p});var n=r(7462),o=(r(7294),r(3905));const c={},a="Secret",s={unversionedId:"concepts/secret",id:"concepts/secret",title:"Secret",description:"A lot of transformation operations require credentials to execute. These credentials (secrets) are needed in some tasks,",source:"@site/docs/concepts/secret.md",sourceDirName:"concepts",slug:"/concepts/secret",permalink:"/optimus/docs/concepts/secret",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/secret.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Intervals and Windows",permalink:"/optimus/docs/concepts/intervals-and-windows"},next:{title:"Plugin",permalink:"/optimus/docs/concepts/plugin"}},i={},p=[],l={toc:p},u="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(u,(0,n.Z)({},l,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"secret"},"Secret"),(0,o.kt)("p",null,"A lot of transformation operations require credentials to execute. These credentials (secrets) are needed in some tasks,\nhooks, and may also be needed in deployment processes such as dependency resolution. Optimus provides a convenient way\nto store secrets and make them accessible in containers during the execution."),(0,o.kt)("p",null,"You can easily create, update, and delete your own secrets using CLI or REST API. Secrets can be created at a project\nlevel which is accessible from all the namespaces in the project, or can just be created at the namespace level. These\nsecrets will then can be used as part of the job spec configuration using macros with their names. Only the secrets\ncreated at the project & namespace the job belongs to can be referenced."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[2147],{3905:(e,t,r)=>{r.d(t,{Zo:()=>l,kt:()=>f});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function c(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function a(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var c=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var i=n.createContext({}),p=function(e){var t=n.useContext(i),r=t;return e&&(r="function"==typeof e?e(t):a(a({},t),e)),r},l=function(e){var t=p(e.components);return n.createElement(i.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,c=e.originalType,i=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),u=p(r),m=o,f=u["".concat(i,".").concat(m)]||u[m]||d[m]||c;return r?n.createElement(f,a(a({ref:t},l),{},{components:r})):n.createElement(f,a({ref:t},l))}));function f(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var c=r.length,a=new Array(c);a[0]=m;var s={};for(var i in t)hasOwnProperty.call(t,i)&&(s[i]=t[i]);s.originalType=e,s[u]="string"==typeof e?e:o,a[1]=s;for(var p=2;p{r.r(t),r.d(t,{assets:()=>i,contentTitle:()=>a,default:()=>d,frontMatter:()=>c,metadata:()=>s,toc:()=>p});var n=r(7462),o=(r(7294),r(3905));const c={},a="Secret",s={unversionedId:"concepts/secret",id:"concepts/secret",title:"Secret",description:"A lot of transformation operations require credentials to execute. These credentials (secrets) are needed in some tasks,",source:"@site/docs/concepts/secret.md",sourceDirName:"concepts",slug:"/concepts/secret",permalink:"/optimus/docs/concepts/secret",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/secret.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Intervals and Windows",permalink:"/optimus/docs/concepts/intervals-and-windows"},next:{title:"Plugin",permalink:"/optimus/docs/concepts/plugin"}},i={},p=[],l={toc:p},u="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(u,(0,n.Z)({},l,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"secret"},"Secret"),(0,o.kt)("p",null,"A lot of transformation operations require credentials to execute. These credentials (secrets) are needed in some tasks,\nhooks, and may also be needed in deployment processes such as dependency resolution. Optimus provides a convenient way\nto store secrets and make them accessible in containers during the execution."),(0,o.kt)("p",null,"You can easily create, update, and delete your own secrets using CLI or REST API. Secrets can be created at a project\nlevel which is accessible from all the namespaces in the project, or can just be created at the namespace level. These\nsecrets will then can be used as part of the job spec configuration using macros with their names. Only the secrets\ncreated at the project & namespace the job belongs to can be referenced."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/d194c8d1.23a61ae6.js b/assets/js/d194c8d1.fef87132.js similarity index 98% rename from assets/js/d194c8d1.23a61ae6.js rename to assets/js/d194c8d1.fef87132.js index 61a141053c..2b15651001 100644 --- a/assets/js/d194c8d1.23a61ae6.js +++ b/assets/js/d194c8d1.fef87132.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8462],{3905:(e,t,n)=>{n.d(t,{Zo:()=>c,kt:()=>m});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=r.createContext({}),p=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},c=function(e){var t=p(e.components);return r.createElement(l.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},h=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,l=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),u=p(n),h=o,m=u["".concat(l,".").concat(h)]||u[h]||d[h]||a;return n?r.createElement(m,s(s({ref:t},c),{},{components:n})):r.createElement(m,s({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,s=new Array(a);s[0]=h;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i[u]="string"==typeof e?e:o,s[1]=i;for(var p=2;p{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>s,default:()=>d,frontMatter:()=>a,metadata:()=>i,toc:()=>p});var r=n(7462),o=(n(7294),n(3905));const a={},s=void 0,i={unversionedId:"rfcs/support_for_depending_on_external_sensors",id:"rfcs/support_for_depending_on_external_sensors",title:"support_for_depending_on_external_sensors",description:"- Feature Name: Support For Depndening on External Sources",source:"@site/docs/rfcs/20220123_support_for_depending_on_external_sensors.md",sourceDirName:"rfcs",slug:"/rfcs/support_for_depending_on_external_sensors",permalink:"/optimus/docs/rfcs/support_for_depending_on_external_sensors",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220123_support_for_depending_on_external_sensors.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220123,frontMatter:{}},l={},p=[{value:"Http Sensor",id:"http-sensor",level:4},{value:"BQ Sensor",id:"bq-sensor",level:4},{value:"GCS Sensor",id:"gcs-sensor",level:3}],c={toc:p},u="wrapper";function d(e){let{components:t,...n}=e;return(0,o.kt)(u,(0,r.Z)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Feature Name: Support For Depndening on External Sources"),(0,o.kt)("li",{parentName:"ul"},"Status: Draft"),(0,o.kt)("li",{parentName:"ul"},"Start Date: 2022-01-23"),(0,o.kt)("li",{parentName:"ul"},"Authors: ")),(0,o.kt)("h1",{id:"summary"},"Summary"),(0,o.kt)("p",null,"Optimus supports job dependencies, but there is a need for optimus jobs to depend on external sources which are not managed by the optimus server. For example, depending the BQ or GCS data availability or data being managed by another optimus server. Whatever data sources optimus is managing lets have sensors for basic data availability check, in GCS checking for file exists & in BQ taking a select query & returning success when rowcount > 0. For other requirements let's have a http sensor."),(0,o.kt)("h1",{id:"technical-design"},"Technical Design"),(0,o.kt)("p",null,"Optimus can add support for all the sensors as libraries, which will be evaulated within the execution envrionment of the user, all variables will be returned for a given scheduled date through the api call which will be used by the actual sensor execution. "),(0,o.kt)("p",null,"Optimus provides libraries needed for the above operations which can be used in the respective execution environment of the scheduler, currently the library will be offered in python."),(0,o.kt)("p",null,"The ",(0,o.kt)("inlineCode",{parentName:"p"},"/intance")," api call can accept params to filter what to return to just reduce the unnecessary payload & return only the needed variables, as sensors execute a lot."),(0,o.kt)("h4",{id:"http-sensor"},(0,o.kt)("strong",{parentName:"h4"},"Http Sensor")),(0,o.kt)("p",null,"If the call returns 200 then the sensor succeeds"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"dependencies : \n type : http\n endpoint : url\n headers :\n body :\n \n")),(0,o.kt)("h4",{id:"bq-sensor"},"BQ Sensor"),(0,o.kt)("p",null,"If the query results in rows then the sensor succeeds"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"dependencies : \n type : bq\n query : \n service_account :\n \n")),(0,o.kt)("h3",{id:"gcs-sensor"},"GCS Sensor"),(0,o.kt)("p",null,"If the path exists then the sensor succeeds"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"dependencies : \n type : gcs\n path : \n service_account : \n")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8462],{3905:(e,t,n)=>{n.d(t,{Zo:()=>c,kt:()=>m});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function s(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var l=r.createContext({}),p=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):s(s({},t),e)),n},c=function(e){var t=p(e.components);return r.createElement(l.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},h=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,l=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),u=p(n),h=o,m=u["".concat(l,".").concat(h)]||u[h]||d[h]||a;return n?r.createElement(m,s(s({ref:t},c),{},{components:n})):r.createElement(m,s({ref:t},c))}));function m(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,s=new Array(a);s[0]=h;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i[u]="string"==typeof e?e:o,s[1]=i;for(var p=2;p{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>s,default:()=>d,frontMatter:()=>a,metadata:()=>i,toc:()=>p});var r=n(7462),o=(n(7294),n(3905));const a={},s=void 0,i={unversionedId:"rfcs/support_for_depending_on_external_sensors",id:"rfcs/support_for_depending_on_external_sensors",title:"support_for_depending_on_external_sensors",description:"- Feature Name: Support For Depndening on External Sources",source:"@site/docs/rfcs/20220123_support_for_depending_on_external_sensors.md",sourceDirName:"rfcs",slug:"/rfcs/support_for_depending_on_external_sensors",permalink:"/optimus/docs/rfcs/support_for_depending_on_external_sensors",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220123_support_for_depending_on_external_sensors.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220123,frontMatter:{}},l={},p=[{value:"Http Sensor",id:"http-sensor",level:4},{value:"BQ Sensor",id:"bq-sensor",level:4},{value:"GCS Sensor",id:"gcs-sensor",level:3}],c={toc:p},u="wrapper";function d(e){let{components:t,...n}=e;return(0,o.kt)(u,(0,r.Z)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("ul",null,(0,o.kt)("li",{parentName:"ul"},"Feature Name: Support For Depndening on External Sources"),(0,o.kt)("li",{parentName:"ul"},"Status: Draft"),(0,o.kt)("li",{parentName:"ul"},"Start Date: 2022-01-23"),(0,o.kt)("li",{parentName:"ul"},"Authors: ")),(0,o.kt)("h1",{id:"summary"},"Summary"),(0,o.kt)("p",null,"Optimus supports job dependencies, but there is a need for optimus jobs to depend on external sources which are not managed by the optimus server. For example, depending the BQ or GCS data availability or data being managed by another optimus server. Whatever data sources optimus is managing lets have sensors for basic data availability check, in GCS checking for file exists & in BQ taking a select query & returning success when rowcount > 0. For other requirements let's have a http sensor."),(0,o.kt)("h1",{id:"technical-design"},"Technical Design"),(0,o.kt)("p",null,"Optimus can add support for all the sensors as libraries, which will be evaulated within the execution envrionment of the user, all variables will be returned for a given scheduled date through the api call which will be used by the actual sensor execution. "),(0,o.kt)("p",null,"Optimus provides libraries needed for the above operations which can be used in the respective execution environment of the scheduler, currently the library will be offered in python."),(0,o.kt)("p",null,"The ",(0,o.kt)("inlineCode",{parentName:"p"},"/intance")," api call can accept params to filter what to return to just reduce the unnecessary payload & return only the needed variables, as sensors execute a lot."),(0,o.kt)("h4",{id:"http-sensor"},(0,o.kt)("strong",{parentName:"h4"},"Http Sensor")),(0,o.kt)("p",null,"If the call returns 200 then the sensor succeeds"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"dependencies : \n type : http\n endpoint : url\n headers :\n body :\n \n")),(0,o.kt)("h4",{id:"bq-sensor"},"BQ Sensor"),(0,o.kt)("p",null,"If the query results in rows then the sensor succeeds"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"dependencies : \n type : bq\n query : \n service_account :\n \n")),(0,o.kt)("h3",{id:"gcs-sensor"},"GCS Sensor"),(0,o.kt)("p",null,"If the path exists then the sensor succeeds"),(0,o.kt)("pre",null,(0,o.kt)("code",{parentName:"pre",className:"language-yaml"},"dependencies : \n type : gcs\n path : \n service_account : \n")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/d4c33874.7158cdd1.js b/assets/js/d4c33874.1c5e822c.js similarity index 98% rename from assets/js/d4c33874.7158cdd1.js rename to assets/js/d4c33874.1c5e822c.js index 236bbbe9bc..3ce50e5e47 100644 --- a/assets/js/d4c33874.7158cdd1.js +++ b/assets/js/d4c33874.1c5e822c.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8124],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>b});var i=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function o(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var l=i.createContext({}),c=function(e){var t=i.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},p=function(e){var t=c(e.components);return i.createElement(l.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},m=i.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=c(n),m=r,b=u["".concat(l,".").concat(m)]||u[m]||d[m]||a;return n?i.createElement(b,o(o({ref:t},p),{},{components:n})):i.createElement(b,o({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,o=new Array(a);o[0]=m;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[u]="string"==typeof e?e:r,o[1]=s;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>o,default:()=>d,frontMatter:()=>a,metadata:()=>s,toc:()=>c});var i=n(7462),r=(n(7294),n(3905));const a={},o="Verifying the Jobs",s={unversionedId:"client-guide/verifying-jobs",id:"client-guide/verifying-jobs",title:"Verifying the Jobs",description:"Minimize the chances of having the job failed in runtime by validating and inspecting it before deployment.",source:"@site/docs/client-guide/verifying-jobs.md",sourceDirName:"client-guide",slug:"/client-guide/verifying-jobs",permalink:"/optimus/docs/client-guide/verifying-jobs",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/verifying-jobs.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Setting up Alert to Job",permalink:"/optimus/docs/client-guide/setting-up-alert"},next:{title:"Applying Job Specifications",permalink:"/optimus/docs/client-guide/applying-job-specifications"}},l={},c=[{value:"Validate Jobs",id:"validate-jobs",level:2},{value:"Inspect Job",id:"inspect-job",level:2}],p={toc:c},u="wrapper";function d(e){let{components:t,...n}=e;return(0,r.kt)(u,(0,i.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"verifying-the-jobs"},"Verifying the Jobs"),(0,r.kt)("p",null,"Minimize the chances of having the job failed in runtime by validating and inspecting it before deployment."),(0,r.kt)("h2",{id:"validate-jobs"},"Validate Jobs"),(0,r.kt)("p",null,"Job validation is being done per namespace. Try it out by running this command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job validate --namespace sample_namespace --verbose\n")),(0,r.kt)("p",null,"Make sure you are running the above command in the same directory as where your client configuration (optimus.yaml)\nis located. Or if not, you can provide the command by adding a config flag."),(0,r.kt)("p",null,"By running the above command, Optimus CLI will try to fetch all of the jobs under sample_namespace\u2019s job path that\nhas been specified in the client configuration. The verbose flag will be helpful to print out the jobs being processed.\nAny jobs that have missing mandatory configuration, contain an invalid query, or cause cyclic dependency will be pointed out."),(0,r.kt)("h2",{id:"inspect-job"},"Inspect Job"),(0,r.kt)("p",null,"You can try to inspect a single job, for example checking what are the upstream/dependencies, does it has any downstream,\nor whether it has any warnings. This inspect command can be done against a job that has been registered or not registered\nto your Optimus server."),(0,r.kt)("p",null,"To inspect a job in your local:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job inspect \n")),(0,r.kt)("p",null,"To inspect a job in the server:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job inspect --server\n")),(0,r.kt)("p",null,"You will find mainly 3 sections for inspection:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"Basic Info"),":\nOptimus will print the job\u2019s specification, including the resource destination & sources (if any), and whether it has any soft warning."),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"Upstreams"),":\nWill prints what are the jobs that this job depends on. Do notice there might be internal upstreams, external (cross-server) upstreams, HTTP upstreams, and unknown upstreams (not registered in Optimus)."),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"Downstreams"),":\nWill prints what are the jobs that depends on this job.")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8124],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>b});var i=n(7294);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,i)}return n}function o(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var l=i.createContext({}),c=function(e){var t=i.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},p=function(e){var t=c(e.components);return i.createElement(l.Provider,{value:t},e.children)},u="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},m=i.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),u=c(n),m=r,b=u["".concat(l,".").concat(m)]||u[m]||d[m]||a;return n?i.createElement(b,o(o({ref:t},p),{},{components:n})):i.createElement(b,o({ref:t},p))}));function b(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,o=new Array(a);o[0]=m;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s[u]="string"==typeof e?e:r,o[1]=s;for(var c=2;c{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>o,default:()=>d,frontMatter:()=>a,metadata:()=>s,toc:()=>c});var i=n(7462),r=(n(7294),n(3905));const a={},o="Verifying the Jobs",s={unversionedId:"client-guide/verifying-jobs",id:"client-guide/verifying-jobs",title:"Verifying the Jobs",description:"Minimize the chances of having the job failed in runtime by validating and inspecting it before deployment.",source:"@site/docs/client-guide/verifying-jobs.md",sourceDirName:"client-guide",slug:"/client-guide/verifying-jobs",permalink:"/optimus/docs/client-guide/verifying-jobs",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/verifying-jobs.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Setting up Alert to Job",permalink:"/optimus/docs/client-guide/setting-up-alert"},next:{title:"Applying Job Specifications",permalink:"/optimus/docs/client-guide/applying-job-specifications"}},l={},c=[{value:"Validate Jobs",id:"validate-jobs",level:2},{value:"Inspect Job",id:"inspect-job",level:2}],p={toc:c},u="wrapper";function d(e){let{components:t,...n}=e;return(0,r.kt)(u,(0,i.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"verifying-the-jobs"},"Verifying the Jobs"),(0,r.kt)("p",null,"Minimize the chances of having the job failed in runtime by validating and inspecting it before deployment."),(0,r.kt)("h2",{id:"validate-jobs"},"Validate Jobs"),(0,r.kt)("p",null,"Job validation is being done per namespace. Try it out by running this command:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job validate --namespace sample_namespace --verbose\n")),(0,r.kt)("p",null,"Make sure you are running the above command in the same directory as where your client configuration (optimus.yaml)\nis located. Or if not, you can provide the command by adding a config flag."),(0,r.kt)("p",null,"By running the above command, Optimus CLI will try to fetch all of the jobs under sample_namespace\u2019s job path that\nhas been specified in the client configuration. The verbose flag will be helpful to print out the jobs being processed.\nAny jobs that have missing mandatory configuration, contain an invalid query, or cause cyclic dependency will be pointed out."),(0,r.kt)("h2",{id:"inspect-job"},"Inspect Job"),(0,r.kt)("p",null,"You can try to inspect a single job, for example checking what are the upstream/dependencies, does it has any downstream,\nor whether it has any warnings. This inspect command can be done against a job that has been registered or not registered\nto your Optimus server."),(0,r.kt)("p",null,"To inspect a job in your local:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job inspect \n")),(0,r.kt)("p",null,"To inspect a job in the server:"),(0,r.kt)("pre",null,(0,r.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus job inspect --server\n")),(0,r.kt)("p",null,"You will find mainly 3 sections for inspection:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"Basic Info"),":\nOptimus will print the job\u2019s specification, including the resource destination & sources (if any), and whether it has any soft warning."),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"Upstreams"),":\nWill prints what are the jobs that this job depends on. Do notice there might be internal upstreams, external (cross-server) upstreams, HTTP upstreams, and unknown upstreams (not registered in Optimus)."),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("strong",{parentName:"li"},"Downstreams"),":\nWill prints what are the jobs that depends on this job.")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/d6ec5803.178dc964.js b/assets/js/d6ec5803.0fcb8587.js similarity index 99% rename from assets/js/d6ec5803.178dc964.js rename to assets/js/d6ec5803.0fcb8587.js index b412647855..7a6055b52c 100644 --- a/assets/js/d6ec5803.178dc964.js +++ b/assets/js/d6ec5803.0fcb8587.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[7997],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>g});var a=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function o(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},m=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,r=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=p(n),m=i,g=c["".concat(s,".").concat(m)]||c[m]||d[m]||r;return n?a.createElement(g,o(o({ref:t},u),{},{components:n})):a.createElement(g,o({ref:t},u))}));function g(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var r=n.length,o=new Array(r);o[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[c]="string"==typeof e?e:i,o[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>o,default:()=>d,frontMatter:()=>r,metadata:()=>l,toc:()=>p});var a=n(7462),i=(n(7294),n(3905));const r={},o="Tutorial of Plugin Development",l={unversionedId:"building-plugin/tutorial",id:"building-plugin/tutorial",title:"Tutorial of Plugin Development",description:"To demonstrate the previous-mentioned wrapping functionality, let's create a plugin in Go as well as a yaml definition",source:"@site/docs/building-plugin/tutorial.md",sourceDirName:"building-plugin",slug:"/building-plugin/tutorial",permalink:"/optimus/docs/building-plugin/tutorial",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/building-plugin/tutorial.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Introduction of Plugin Development",permalink:"/optimus/docs/building-plugin/introduction"},next:{title:"Contributing",permalink:"/optimus/docs/contribute/contribution-process"}},s={},p=[{value:"Preparing task executor",id:"preparing-task-executor",level:2},{value:"Creating a yaml plugin",id:"creating-a-yaml-plugin",level:2},{value:"How to Use",id:"how-to-use",level:2},{value:"Installing the plugin in server",id:"installing-the-plugin-in-server",level:3},{value:"Installing the plugin in client",id:"installing-the-plugin-in-client",level:3},{value:"Use the plugin in job creation",id:"use-the-plugin-in-job-creation",level:3}],u={toc:p},c="wrapper";function d(e){let{components:t,...n}=e;return(0,i.kt)(c,(0,a.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"tutorial-of-plugin-development"},"Tutorial of Plugin Development"),(0,i.kt)("p",null,"To demonstrate the previous-mentioned wrapping functionality, let's create a plugin in Go as well as a yaml definition\nand use python for actual transformation logic. You can choose to fork this ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/kushsharma/optimus-plugins"},"example"),"\ntemplate and modify it as per your needs or start fresh. To demonstrate how to start from scratch, we will be starting\nfrom an empty git repository and build a plugin which will find potential hazardous ",(0,i.kt)("strong",{parentName:"p"},"Near Earth Orbit")," objects every\nday, let's call it ",(0,i.kt)("strong",{parentName:"p"},"neo")," for short."),(0,i.kt)("p",null,"Brief description of Neo is as follows"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Using NASA ",(0,i.kt)("a",{parentName:"li",href:"https://api.nasa.gov/"},"API")," we can get count of hazardous objects, their diameter and velocity."),(0,i.kt)("li",{parentName:"ul"},"Task will need two config as input, RANGE_START, RANGE_END as date time string which will filter the count for\nthis specific period only."),(0,i.kt)("li",{parentName:"ul"},"Execute every day, say at 2 AM."),(0,i.kt)("li",{parentName:"ul"},"Need a secret token that will be passed to NASA api endpoint for each request."),(0,i.kt)("li",{parentName:"ul"},"Output of this object count can be printed in logs for now but in a real use case can be pushed to Kafka topic or\nwritten to a database."),(0,i.kt)("li",{parentName:"ul"},"Plugin will be written in ",(0,i.kt)("strong",{parentName:"li"},"YAML")," format and Neo in ",(0,i.kt)("strong",{parentName:"li"},"python"),".")),(0,i.kt)("h2",{id:"preparing-task-executor"},"Preparing task executor"),(0,i.kt)("p",null,"Start by initialising an empty git repository with the following folder structure"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},".git\n/task\n /neo\n /executor\n /main.py\n /requirements.txt\n /Dockerfile\nREADME.md\n")),(0,i.kt)("p",null,"That is three folders one inside another. This might look confusing for now, a lot of things will, but just keep going.\nCreate an empty python file in executor main.py, this is where the main logic for interacting with nasa api and\ngenerating output will be. For simplicity, lets use as minimal things as possible."),(0,i.kt)("p",null,"Add the following code to main.py"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-python"},'import os\nimport requests\nimport json\n\ndef start():\n """\n Sends a http call to nasa api, parses response and prints potential hazardous\n objects in near earth orbit\n :return:\n """\n opt_config = fetch_config_from_optimus()\n\n # user configuration for date range\n range_start = opt_config["envs"]["RANGE_START"]\n range_end = opt_config["envs"]["RANGE_END"]\n\n secret_key = os.environ["SECRET_KEY"]\n\n # secret token required for NASA API being passed using job spec\n api_key = json.loads(secret_key)\n if api_key is None:\n raise Exception("invalid api token")\n\n # send the request for given date range\n r = requests.get(url="https://api.nasa.gov/neo/rest/v1/feed",\n params={\'start_date\': range_start, \'end_date\': range_end, \'api_key\': api_key})\n\n # extracting data in json format\n print("for date range {} - {}".format(range_start, range_end))\n print_details(r.json())\n\n return\n \n\ndef fetch_config_from_optimus():\n """\n Fetch configuration inputs required to run this task for a single schedule day\n Configurations are fetched using optimus rest api\n :return:\n """\n # try printing os env to see what all we have for debugging\n # print(os.environ)\n\n # prepare request\n optimus_host = os.environ["OPTIMUS_HOSTNAME"]\n scheduled_at = os.environ["SCHEDULED_AT"]\n project_name = os.environ["PROJECT"]\n job_name = os.environ["JOB_NAME"]\n\n r = requests.post(url="http://{}/api/v1/project/{}/job/{}/instance".format(optimus_host, project_name, job_name),\n json={\'scheduled_at\': scheduled_at,\n \'instance_name\': "neo",\n \'instance_type\': "TASK"})\n instance = r.json()\n\n # print(instance)\n return instance["context"]\n \n \n \nif __name__ == "__main__":\n start()\n\n')),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"api_key")," is a token provided by nasa during registration. This token will be passed as a parameter in each http call.\n",(0,i.kt)("strong",{parentName:"p"},"SECRET_PATH")," is the path to a file which will contain this token in json and will be mounted inside the docker\ncontainer by Optimus."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"start")," function is using ",(0,i.kt)("strong",{parentName:"p"},"fetch_config_from_optimus()")," to get the date range for which this task executes for\nan iteration. In this example, configuration is fetched using REST APIs provided by optimus although there are variety\nof ways to get them. After extracting ",(0,i.kt)("strong",{parentName:"p"},"API_KEY")," from secret file, unmarshalling it to json with ",(0,i.kt)("strong",{parentName:"p"},"json.load()"),"\nsend a http request to nasa api. Response can be parsed and printed using the following function:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-python"},'def print_details(jd):\n """\n Parse and calculate what we need from NASA endpoint response\n\n :param jd: json data fetched from NASA API\n :return:\n """\n element_count = jd[\'element_count\']\n potentially_hazardous = []\n for search_date in jd[\'near_earth_objects\'].keys():\n for neo in jd[\'near_earth_objects\'][search_date]:\n if neo["is_potentially_hazardous_asteroid"] is True:\n potentially_hazardous.append({\n "name": neo["name"],\n "estimated_diameter_km": neo["estimated_diameter"]["kilometers"]["estimated_diameter_max"],\n "relative_velocity_kmh": neo["close_approach_data"][0]["relative_velocity"]["kilometers_per_hour"]\n })\n\n print("total tracking: {}\\npotential hazardous: {}".format(element_count, len(potentially_hazardous)))\n for haz in potentially_hazardous:\n print("Name: {}\\nEstimated Diameter: {} km\\nRelative Velocity: {} km/h\\n\\n".format(\n haz["name"],\n haz["estimated_diameter_km"],\n haz["relative_velocity_kmh"]\n ))\n return\n\n')),(0,i.kt)("p",null,"Finish it off by adding the main function"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-python"},'if __name__ == "__main__":\nstart()\n')),(0,i.kt)("p",null,"Add requests library in requirements.txt"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"requests==v2.25.1\n")),(0,i.kt)("p",null,"Once the python code is ready, wrap this in a Dockerfile"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-dockerfile"},'# set base image (host OS)\nFROM python:3.8\n\n# set the working directory in the container\nRUN mkdir -p /opt\nWORKDIR /opt\n\n# copy the content of the local src directory to the working directory\nCOPY task/neo/executor .\n\n# install dependencies\nRUN pip install -r requirements.txt\n\nCMD ["python3", "main.py"]\n\n')),(0,i.kt)("h2",{id:"creating-a-yaml-plugin"},"Creating a yaml plugin"),(0,i.kt)("p",null,"The Yaml implementation is as simple as providing the plugin details as below."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},"name: Neo\ndescription: Near earth object tracker\nplugintype: task\npluginversion: latest\nimage: ghcr.io/kushsharma/optimus-task-neo-executor\nsecretpath: /tmp/.secrets\n\nquestions:\n- name: RANGE_START\n prompt: Date range start\n help: YYYY-MM-DD format\n required: true\n- name: RANGE_END\n prompt: Date range end\n help: YYYY-MM-DD format\n required: true\n")),(0,i.kt)("p",null,"Based on the usecase, additional validation can be added to the questions section."),(0,i.kt)("p",null,"This yaml plugin can be placed anywhere, however for this tutorial let\u2019s place it under ",(0,i.kt)("inlineCode",{parentName:"p"},"../task/neo")," directory and\nname it as ",(0,i.kt)("inlineCode",{parentName:"p"},"optimus-plugin-neo.yaml"),"."),(0,i.kt)("p",null,"Note: As part of this tutorial, we are not providing binary plugin tutorial as it is going to be deprecated. "),(0,i.kt)("h2",{id:"how-to-use"},"How to Use"),(0,i.kt)("p",null,"Before using, let\u2019s install this new plugin in server and client."),(0,i.kt)("h3",{id:"installing-the-plugin-in-server"},"Installing the plugin in server"),(0,i.kt)("p",null,"To use the created plugin in your server, you can simpy add the plugin path in the server config:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},"plugin:\n artifacts:\n - ../task/neo/optimus-plugin-neo.yaml\n")),(0,i.kt)("p",null,"To apply the change, you can follow either of these options:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Start Optimus server using ",(0,i.kt)("inlineCode",{parentName:"li"},"--install-plugins")," flag, or"),(0,i.kt)("li",{parentName:"ul"},"Install the plugin separately before starting the server using ",(0,i.kt)("inlineCode",{parentName:"li"},"optimus plugin install")," command.")),(0,i.kt)("p",null,(0,i.kt)("em",{parentName:"p"},"Note: Take a look at installing plugins in server ",(0,i.kt)("a",{parentName:"em",href:"/optimus/docs/server-guide/installing-plugins"},"guide")," for more information.")),(0,i.kt)("h3",{id:"installing-the-plugin-in-client"},"Installing the plugin in client"),(0,i.kt)("p",null,"Install the plugin in client side by syncing it from server using below command."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus plugin sync\n")),(0,i.kt)("p",null,"Once finished, the ",(0,i.kt)("inlineCode",{parentName:"p"},"Neo")," plugin will be available in the ",(0,i.kt)("inlineCode",{parentName:"p"},".plugins")," directory."),(0,i.kt)("h3",{id:"use-the-plugin-in-job-creation"},"Use the plugin in job creation"),(0,i.kt)("p",null,"Once everything is built and in place, we can generate job specifications that uses neo as the task type."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"optimus create job\n? What is the job name? is_last_day_on_earth\n? Who is the owner of this job? owner@example.io\n? Which task to run? neo\n? Specify the start date 2022-01-25\n? Specify the interval (in crontab notation) 0 2 * * *\n? Window truncate to: d\n? Window offset: 0\n? Window size: 24h\n...\njob created successfully is_last_day_on_earth\n")),(0,i.kt)("p",null,"Create a commit and deploy this specification if you are using optimus with a git managed repositories or send\na REST call or use GRPC, whatever floats your boat."),(0,i.kt)("p",null,"Once the job has been deployed and run, open the task log and verify something like this"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"total tracking: 14\npotential hazardous: 1\nName: (2014 KP4)\nEstimated Diameter: 0.8204270649 km\nRelative Velocity: 147052.9914506647 km/h\n")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[7997],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>g});var a=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function o(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},m=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,r=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=p(n),m=i,g=c["".concat(s,".").concat(m)]||c[m]||d[m]||r;return n?a.createElement(g,o(o({ref:t},u),{},{components:n})):a.createElement(g,o({ref:t},u))}));function g(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var r=n.length,o=new Array(r);o[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[c]="string"==typeof e?e:i,o[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>o,default:()=>d,frontMatter:()=>r,metadata:()=>l,toc:()=>p});var a=n(7462),i=(n(7294),n(3905));const r={},o="Tutorial of Plugin Development",l={unversionedId:"building-plugin/tutorial",id:"building-plugin/tutorial",title:"Tutorial of Plugin Development",description:"To demonstrate the previous-mentioned wrapping functionality, let's create a plugin in Go as well as a yaml definition",source:"@site/docs/building-plugin/tutorial.md",sourceDirName:"building-plugin",slug:"/building-plugin/tutorial",permalink:"/optimus/docs/building-plugin/tutorial",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/building-plugin/tutorial.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Introduction of Plugin Development",permalink:"/optimus/docs/building-plugin/introduction"},next:{title:"Contributing",permalink:"/optimus/docs/contribute/contribution-process"}},s={},p=[{value:"Preparing task executor",id:"preparing-task-executor",level:2},{value:"Creating a yaml plugin",id:"creating-a-yaml-plugin",level:2},{value:"How to Use",id:"how-to-use",level:2},{value:"Installing the plugin in server",id:"installing-the-plugin-in-server",level:3},{value:"Installing the plugin in client",id:"installing-the-plugin-in-client",level:3},{value:"Use the plugin in job creation",id:"use-the-plugin-in-job-creation",level:3}],u={toc:p},c="wrapper";function d(e){let{components:t,...n}=e;return(0,i.kt)(c,(0,a.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("h1",{id:"tutorial-of-plugin-development"},"Tutorial of Plugin Development"),(0,i.kt)("p",null,"To demonstrate the previous-mentioned wrapping functionality, let's create a plugin in Go as well as a yaml definition\nand use python for actual transformation logic. You can choose to fork this ",(0,i.kt)("a",{parentName:"p",href:"https://github.com/kushsharma/optimus-plugins"},"example"),"\ntemplate and modify it as per your needs or start fresh. To demonstrate how to start from scratch, we will be starting\nfrom an empty git repository and build a plugin which will find potential hazardous ",(0,i.kt)("strong",{parentName:"p"},"Near Earth Orbit")," objects every\nday, let's call it ",(0,i.kt)("strong",{parentName:"p"},"neo")," for short."),(0,i.kt)("p",null,"Brief description of Neo is as follows"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Using NASA ",(0,i.kt)("a",{parentName:"li",href:"https://api.nasa.gov/"},"API")," we can get count of hazardous objects, their diameter and velocity."),(0,i.kt)("li",{parentName:"ul"},"Task will need two config as input, RANGE_START, RANGE_END as date time string which will filter the count for\nthis specific period only."),(0,i.kt)("li",{parentName:"ul"},"Execute every day, say at 2 AM."),(0,i.kt)("li",{parentName:"ul"},"Need a secret token that will be passed to NASA api endpoint for each request."),(0,i.kt)("li",{parentName:"ul"},"Output of this object count can be printed in logs for now but in a real use case can be pushed to Kafka topic or\nwritten to a database."),(0,i.kt)("li",{parentName:"ul"},"Plugin will be written in ",(0,i.kt)("strong",{parentName:"li"},"YAML")," format and Neo in ",(0,i.kt)("strong",{parentName:"li"},"python"),".")),(0,i.kt)("h2",{id:"preparing-task-executor"},"Preparing task executor"),(0,i.kt)("p",null,"Start by initialising an empty git repository with the following folder structure"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},".git\n/task\n /neo\n /executor\n /main.py\n /requirements.txt\n /Dockerfile\nREADME.md\n")),(0,i.kt)("p",null,"That is three folders one inside another. This might look confusing for now, a lot of things will, but just keep going.\nCreate an empty python file in executor main.py, this is where the main logic for interacting with nasa api and\ngenerating output will be. For simplicity, lets use as minimal things as possible."),(0,i.kt)("p",null,"Add the following code to main.py"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-python"},'import os\nimport requests\nimport json\n\ndef start():\n """\n Sends a http call to nasa api, parses response and prints potential hazardous\n objects in near earth orbit\n :return:\n """\n opt_config = fetch_config_from_optimus()\n\n # user configuration for date range\n range_start = opt_config["envs"]["RANGE_START"]\n range_end = opt_config["envs"]["RANGE_END"]\n\n secret_key = os.environ["SECRET_KEY"]\n\n # secret token required for NASA API being passed using job spec\n api_key = json.loads(secret_key)\n if api_key is None:\n raise Exception("invalid api token")\n\n # send the request for given date range\n r = requests.get(url="https://api.nasa.gov/neo/rest/v1/feed",\n params={\'start_date\': range_start, \'end_date\': range_end, \'api_key\': api_key})\n\n # extracting data in json format\n print("for date range {} - {}".format(range_start, range_end))\n print_details(r.json())\n\n return\n \n\ndef fetch_config_from_optimus():\n """\n Fetch configuration inputs required to run this task for a single schedule day\n Configurations are fetched using optimus rest api\n :return:\n """\n # try printing os env to see what all we have for debugging\n # print(os.environ)\n\n # prepare request\n optimus_host = os.environ["OPTIMUS_HOSTNAME"]\n scheduled_at = os.environ["SCHEDULED_AT"]\n project_name = os.environ["PROJECT"]\n job_name = os.environ["JOB_NAME"]\n\n r = requests.post(url="http://{}/api/v1/project/{}/job/{}/instance".format(optimus_host, project_name, job_name),\n json={\'scheduled_at\': scheduled_at,\n \'instance_name\': "neo",\n \'instance_type\': "TASK"})\n instance = r.json()\n\n # print(instance)\n return instance["context"]\n \n \n \nif __name__ == "__main__":\n start()\n\n')),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"api_key")," is a token provided by nasa during registration. This token will be passed as a parameter in each http call.\n",(0,i.kt)("strong",{parentName:"p"},"SECRET_PATH")," is the path to a file which will contain this token in json and will be mounted inside the docker\ncontainer by Optimus."),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"start")," function is using ",(0,i.kt)("strong",{parentName:"p"},"fetch_config_from_optimus()")," to get the date range for which this task executes for\nan iteration. In this example, configuration is fetched using REST APIs provided by optimus although there are variety\nof ways to get them. After extracting ",(0,i.kt)("strong",{parentName:"p"},"API_KEY")," from secret file, unmarshalling it to json with ",(0,i.kt)("strong",{parentName:"p"},"json.load()"),"\nsend a http request to nasa api. Response can be parsed and printed using the following function:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-python"},'def print_details(jd):\n """\n Parse and calculate what we need from NASA endpoint response\n\n :param jd: json data fetched from NASA API\n :return:\n """\n element_count = jd[\'element_count\']\n potentially_hazardous = []\n for search_date in jd[\'near_earth_objects\'].keys():\n for neo in jd[\'near_earth_objects\'][search_date]:\n if neo["is_potentially_hazardous_asteroid"] is True:\n potentially_hazardous.append({\n "name": neo["name"],\n "estimated_diameter_km": neo["estimated_diameter"]["kilometers"]["estimated_diameter_max"],\n "relative_velocity_kmh": neo["close_approach_data"][0]["relative_velocity"]["kilometers_per_hour"]\n })\n\n print("total tracking: {}\\npotential hazardous: {}".format(element_count, len(potentially_hazardous)))\n for haz in potentially_hazardous:\n print("Name: {}\\nEstimated Diameter: {} km\\nRelative Velocity: {} km/h\\n\\n".format(\n haz["name"],\n haz["estimated_diameter_km"],\n haz["relative_velocity_kmh"]\n ))\n return\n\n')),(0,i.kt)("p",null,"Finish it off by adding the main function"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-python"},'if __name__ == "__main__":\nstart()\n')),(0,i.kt)("p",null,"Add requests library in requirements.txt"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"requests==v2.25.1\n")),(0,i.kt)("p",null,"Once the python code is ready, wrap this in a Dockerfile"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-dockerfile"},'# set base image (host OS)\nFROM python:3.8\n\n# set the working directory in the container\nRUN mkdir -p /opt\nWORKDIR /opt\n\n# copy the content of the local src directory to the working directory\nCOPY task/neo/executor .\n\n# install dependencies\nRUN pip install -r requirements.txt\n\nCMD ["python3", "main.py"]\n\n')),(0,i.kt)("h2",{id:"creating-a-yaml-plugin"},"Creating a yaml plugin"),(0,i.kt)("p",null,"The Yaml implementation is as simple as providing the plugin details as below."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},"name: Neo\ndescription: Near earth object tracker\nplugintype: task\npluginversion: latest\nimage: ghcr.io/kushsharma/optimus-task-neo-executor\nsecretpath: /tmp/.secrets\n\nquestions:\n- name: RANGE_START\n prompt: Date range start\n help: YYYY-MM-DD format\n required: true\n- name: RANGE_END\n prompt: Date range end\n help: YYYY-MM-DD format\n required: true\n")),(0,i.kt)("p",null,"Based on the usecase, additional validation can be added to the questions section."),(0,i.kt)("p",null,"This yaml plugin can be placed anywhere, however for this tutorial let\u2019s place it under ",(0,i.kt)("inlineCode",{parentName:"p"},"../task/neo")," directory and\nname it as ",(0,i.kt)("inlineCode",{parentName:"p"},"optimus-plugin-neo.yaml"),"."),(0,i.kt)("p",null,"Note: As part of this tutorial, we are not providing binary plugin tutorial as it is going to be deprecated. "),(0,i.kt)("h2",{id:"how-to-use"},"How to Use"),(0,i.kt)("p",null,"Before using, let\u2019s install this new plugin in server and client."),(0,i.kt)("h3",{id:"installing-the-plugin-in-server"},"Installing the plugin in server"),(0,i.kt)("p",null,"To use the created plugin in your server, you can simpy add the plugin path in the server config:"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},"plugin:\n artifacts:\n - ../task/neo/optimus-plugin-neo.yaml\n")),(0,i.kt)("p",null,"To apply the change, you can follow either of these options:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Start Optimus server using ",(0,i.kt)("inlineCode",{parentName:"li"},"--install-plugins")," flag, or"),(0,i.kt)("li",{parentName:"ul"},"Install the plugin separately before starting the server using ",(0,i.kt)("inlineCode",{parentName:"li"},"optimus plugin install")," command.")),(0,i.kt)("p",null,(0,i.kt)("em",{parentName:"p"},"Note: Take a look at installing plugins in server ",(0,i.kt)("a",{parentName:"em",href:"/optimus/docs/server-guide/installing-plugins"},"guide")," for more information.")),(0,i.kt)("h3",{id:"installing-the-plugin-in-client"},"Installing the plugin in client"),(0,i.kt)("p",null,"Install the plugin in client side by syncing it from server using below command."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus plugin sync\n")),(0,i.kt)("p",null,"Once finished, the ",(0,i.kt)("inlineCode",{parentName:"p"},"Neo")," plugin will be available in the ",(0,i.kt)("inlineCode",{parentName:"p"},".plugins")," directory."),(0,i.kt)("h3",{id:"use-the-plugin-in-job-creation"},"Use the plugin in job creation"),(0,i.kt)("p",null,"Once everything is built and in place, we can generate job specifications that uses neo as the task type."),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-shell"},"optimus create job\n? What is the job name? is_last_day_on_earth\n? Who is the owner of this job? owner@example.io\n? Which task to run? neo\n? Specify the start date 2022-01-25\n? Specify the interval (in crontab notation) 0 2 * * *\n? Window truncate to: d\n? Window offset: 0\n? Window size: 24h\n...\njob created successfully is_last_day_on_earth\n")),(0,i.kt)("p",null,"Create a commit and deploy this specification if you are using optimus with a git managed repositories or send\na REST call or use GRPC, whatever floats your boat."),(0,i.kt)("p",null,"Once the job has been deployed and run, open the task log and verify something like this"),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"total tracking: 14\npotential hazardous: 1\nName: (2014 KP4)\nEstimated Diameter: 0.8204270649 km\nRelative Velocity: 147052.9914506647 km/h\n")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/e22177c2.9fd87071.js b/assets/js/e22177c2.c87118ee.js similarity index 99% rename from assets/js/e22177c2.9fd87071.js rename to assets/js/e22177c2.c87118ee.js index 2b40750231..c3fe6cc7c1 100644 --- a/assets/js/e22177c2.9fd87071.js +++ b/assets/js/e22177c2.c87118ee.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3807],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>g});var a=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},m="mdxType",c={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,r=e.originalType,s=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),m=p(n),d=i,g=m["".concat(s,".").concat(d)]||m[d]||c[d]||r;return n?a.createElement(g,l(l({ref:t},u),{},{components:n})):a.createElement(g,l({ref:t},u))}));function g(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var r=n.length,l=new Array(r);l[0]=d;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o[m]="string"==typeof e?e:i,l[1]=o;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>c,frontMatter:()=>r,metadata:()=>o,toc:()=>p});var a=n(7462),i=(n(7294),n(3905));const r={},l=void 0,o={unversionedId:"rfcs/simplify_plugin_maintenance",id:"rfcs/simplify_plugin_maintenance",title:"simplify_plugin_maintenance",description:"- Feature Name: Simplify Plugins",source:"@site/docs/rfcs/20220507_simplify_plugin_maintenance.md",sourceDirName:"rfcs",slug:"/rfcs/simplify_plugin_maintenance",permalink:"/optimus/docs/rfcs/simplify_plugin_maintenance",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220507_simplify_plugin_maintenance.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220507,frontMatter:{}},s={},p=[{value:"Background :",id:"background-",level:2},{value:"Changes that trigger a new release in Optimus setup:",id:"changes-that-trigger-a-new-release-in-optimus-setup",level:3},{value:"Release dependencies as per current design",id:"release-dependencies-as-per-current-design",level:3},{value:"1. Avoid Wrapping Executor Images :",id:"1-avoid-wrapping-executor-images--",level:3},{value:"2. Simplify Plugin Installation :",id:"2-simplify-plugin-installation-",level:3},{value:"Approach :",id:"approach-",level:2},{value:"1. Avoid Wrapping Executor Images :",id:"1-avoid-wrapping-executor-images---1",level:3},{value:"2. Simplify Plugin Installations :",id:"2-simplify-plugin-installations-",level:3},{value:"A) Plugin Manager:",id:"a-plugin-manager",level:4},{value:"B) Yaml Plugin Interface: (for client side simplification)",id:"b-yaml-plugin-interface-for-client-side-simplification",level:4},{value:"Result:",id:"result",level:2}],u={toc:p},m="wrapper";function c(e){let{components:t,...n}=e;return(0,i.kt)(m,(0,a.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Feature Name: Simplify Plugins"),(0,i.kt)("li",{parentName:"ul"},"Status: Draft"),(0,i.kt)("li",{parentName:"ul"},"Start Date: 2022-05-07"),(0,i.kt)("li",{parentName:"ul"},"Author: Saikumar")),(0,i.kt)("h1",{id:"summary"},"Summary"),(0,i.kt)("p",null,"The scope of this rfc is to simplify the release and deployment operations w.r.t the optimus plugin ecosystem."),(0,i.kt)("p",null,"The proposal here is to :"),(0,i.kt)("ol",null,(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("strong",{parentName:"li"},"Avoid Wrapping Executor Images")," :",(0,i.kt)("br",{parentName:"li"}),"Decouple the executor_boot_process and the executor as separate containers where the airflow worker launches a pod with init-container (for boot process) adjacent to executor container."),(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("strong",{parentName:"li"},"Simplfy Plugin Installation")," :",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"Server end")," : Install plugins declaratively at runtime instead of manually baking them into the optimus server image (in kubernetes setup)."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"Client end")," : Plugin interface for client-end is limited to support Version, Survey Questions and Answers etc. that can be extracted out from the current plugin interface and maintained as yaml file which simplifies platform dependent plugin distribution for cli.")))),(0,i.kt)("h1",{id:"technical-design"},"Technical Design"),(0,i.kt)("h2",{id:"background-"},"Background :"),(0,i.kt)("h3",{id:"changes-that-trigger-a-new-release-in-optimus-setup"},"Changes that trigger a new release in Optimus setup:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Executor Image changes"),(0,i.kt)("li",{parentName:"ul"},"Executor Image Wrapper changes"),(0,i.kt)("li",{parentName:"ul"},"Plugin binary changes"),(0,i.kt)("li",{parentName:"ul"},"Optimus binary changes")),(0,i.kt)("h3",{id:"release-dependencies-as-per-current-design"},"Release dependencies as per current design"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"Executor Image release")," -> ",(0,i.kt)("inlineCode",{parentName:"li"},"Executor Wrapper Image release")," -> ",(0,i.kt)("inlineCode",{parentName:"li"},"Plugin binary release")," -> ",(0,i.kt)("inlineCode",{parentName:"li"},"Server release")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"Plugin binary release")," -> ",(0,i.kt)("inlineCode",{parentName:"li"},"Server release"))),(0,i.kt)("h3",{id:"1-avoid-wrapping-executor-images--"},"1. ",(0,i.kt)("u",null,"Avoid Wrapping Executor Images")," :"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"The ",(0,i.kt)("inlineCode",{parentName:"li"},"executor_boot_process")," and ",(0,i.kt)("inlineCode",{parentName:"li"},"executor")," are coupled:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"-- Plugin repo structure\n/task/\n/task/Dockerfile -- task_image\n/task/executor/Dockerfile -- executor_image\n")),(0,i.kt)("p",null,"Executor Wrapper Image (Task Image) :"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"It's a wrapper around the executor_image to facilitate boot mechanism for executor."),(0,i.kt)("li",{parentName:"ul"},"The optimus binary is downloaded during buildtime of this image."),(0,i.kt)("li",{parentName:"ul"},"During runtime, it does as follow :",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Fetch assets, secrets, env from optimus server."),(0,i.kt)("li",{parentName:"ul"},"Load the env and launches the executor process.")))),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"task_image \n | executor_image\n | optimus-bin\n | entrypoint.sh (load assets, env and launch executor)\n")),(0,i.kt)("p",null,"The ",(0,i.kt)("inlineCode",{parentName:"p"},"optimus-bin")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"entrypoint.sh")," are baked into the ",(0,i.kt)("inlineCode",{parentName:"p"},"task_image")," and is being maintained by task/plugin developers."),(0,i.kt)("h3",{id:"2-simplify-plugin-installation-"},"2. ",(0,i.kt)("u",null,"Simplify Plugin Installation")," :"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Plugin binaries are manually installed (baked into optimus image in kubernetes setup). "),(0,i.kt)("li",{parentName:"ul"},"Any change in plugin code demands re-creation of optimus image with new plugin binary, inturn demanding redeployment of optimus server. (in kubernetes setup)"),(0,i.kt)("li",{parentName:"ul"},"At client side, plugin binaries require support for different platforms.")),(0,i.kt)("h2",{id:"approach-"},"Approach :"),(0,i.kt)("h3",{id:"1-avoid-wrapping-executor-images---1"},"1. ",(0,i.kt)("u",null,"Avoid Wrapping Executor Images ")," :"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Decouple the lifecycle of the executor and the boot process as seperate containers/images.")),(0,i.kt)("img",{src:"images/simplify_plugins_executor.png",alt:"Simplify Plugins Executor",width:"800"}),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Task Boot Sequence"),":"),(0,i.kt)("ol",null,(0,i.kt)("li",{parentName:"ol"},"Airflow worker fetches env and secrets for the job and adds them to the executor pod as environment variables."),(0,i.kt)("li",{parentName:"ol"},"KubernetesPodOperator spawns init-container and executor-container, mounted with shared volume (type emptyDir) for assets."),(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("inlineCode",{parentName:"li"},"init-container")," fetches assets, config, env files and writes onto the shared volume."),(0,i.kt)("li",{parentName:"ol"},"the default entrypoint in the executor-image starts the actual job.")),(0,i.kt)("h3",{id:"2-simplify-plugin-installations-"},"2. ",(0,i.kt)("u",null,"Simplify Plugin Installations")," :"),(0,i.kt)("h4",{id:"a-plugin-manager"},"A) Plugin Manager:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Currently the plugins are maintained as monorepo and versioned together. For any change in a single plugin, a new tar file containing all plugin binaries is created and released."),(0,i.kt)("li",{parentName:"ul"},"A plugin manager is required to support declarative installation of plugins so that plugins can be independently versioned, packaged and installed."),(0,i.kt)("li",{parentName:"ul"},"This plugin manager consumes a config (plugins_config) and downloads artifacts from a plugin repository.")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Optimus support for plugin manager as below.",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"optimus plugin install -c config.yaml")," -- at server"))),(0,i.kt)("li",{parentName:"ul"},"Support for different kinds of plugin repositories (like s3, gcs, url, local file system etc..) gives the added flexibility and options to distribute and install the plugin binaries in different ways."),(0,i.kt)("li",{parentName:"ul"},"Plugins are installed at container runtime and this decouples the building of optimus docker image from plugins installations."),(0,i.kt)("li",{parentName:"ul"},"Example for the plugin_config: ")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},"plugin:\n dir: .plugins\n artifacts:\n # local filesystem for dev\n - ../transformers/dist/bq2bq_darwin_arm64/optimus-bq2bq_darwin_arm64\n # any http uri\n - https://github.com/goto/optimus/releases/download/v0.2.5/optimus_0.2.5_linux_arm64.tar.gz\n \n")),(0,i.kt)("h4",{id:"b-yaml-plugin-interface-for-client-side-simplification"},"B) Yaml Plugin Interface: (for client side simplification)"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Currently plugins are implemented and distributed as binaries and as clients needs to install them, it demands support for different host architectures."),(0,i.kt)("li",{parentName:"ul"},"Since CLI (client side) plugins just require details about plugin such as Version, Suevery Questions etc. the proposal here is to maintain CLI plugins as yaml files."),(0,i.kt)("li",{parentName:"ul"},"Implementation wise, the proposal here is to split the current plugin interface (which only supports interaction with binary plugins) to also accommodate yaml based plugins."),(0,i.kt)("li",{parentName:"ul"},"The above mentioned pluign manager, at server end, would be agnostic about the contents of plugin artifacts from the repository."),(0,i.kt)("li",{parentName:"ul"},"At client side, the CLI could sync the yaml files from the server to stay up-to-date with the server w.r.t plugins."),(0,i.kt)("li",{parentName:"ul"},"At this point, we have the scope to move away from binary plugins except for bq2bq plugin due to its depdendency on ",(0,i.kt)("inlineCode",{parentName:"li"},"ComplileAsset")," and ",(0,i.kt)("inlineCode",{parentName:"li"},"ResolveDependency")," functionalities which are required at server end (not at cli)."),(0,i.kt)("li",{parentName:"ul"},"Handling Bq2Bq plugin:",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Move ",(0,i.kt)("inlineCode",{parentName:"li"},"CompileAsset")," functionality as a part of Bq2bq executor."),(0,i.kt)("li",{parentName:"ul"},"Move ",(0,i.kt)("inlineCode",{parentName:"li"},"ResolveDependency")," functionality to optimus core which should support dependecy-resolution on standard-sql"))),(0,i.kt)("li",{parentName:"ul"},"Meanwhile the Bq2bq plugin is handled, the plugin interface can be maintanined in such a way that it also supports binary plugin in addition to yaml (as backward compaitbility feature)."),(0,i.kt)("li",{parentName:"ul"},"The plugin discovery logic should be to load binary if present, else load yaml file; for a single plugin."),(0,i.kt)("li",{parentName:"ul"},"Now that we have yaml files at server, CLI can sync only the yaml files from the server.",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"optimus plugin sync -c optimus.yaml"))))),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Example representation of the yaml plugin : ")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},'name: bq2bq\ndescription: BigQuery to BigQuery transformation task\nplugintype: task\npluginmods:\n - cli\n - dependencyresolver\npluginversion: 0.1.0-SNAPSHOT-27cb56f\napiversion: []\nimage: docker.io/goto/optimus-task-bq2bq-executor:0.1.0-SNAPSHOT-27cb56f\nsecretpath: /tmp/auth.json\ndependson: []\nhooktype: ""\n\nquestions:\n - name: PROJECT\n prompt: Project ID\n help: Destination bigquery project ID\n regexp: ^[a-zA-Z0-9_\\-]+$\n validationerror: invalid name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)\n minlength: 3\n - name: Dataset\n prompt: Dataset Name\n help: Destination bigquery dataset ID\n regexp: ^[a-zA-Z0-9_\\-]+$\n validationerror: invalid name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)\n minlength: 3\n - name: TABLE\n prompt: Table ID\n help: Destination bigquery table ID\n regexp: ^[a-zA-Z0-9_-]+$\n validationerror: invalid table name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)\n minlength: 3\n maxlength: 1024\n - name: LOAD_METHOD\n prompt: Load method to use on destination\n help: |\n APPEND - Append to existing table\n REPLACE - Deletes existing partition and insert result of select query\n MERGE - DML statements, BQ scripts\n REPLACE_MERGE - [Experimental] Advanced replace using merge query\n default: APPEND\n multiselect:\n - APPEND\n - REPLACE\n - MERGE\n - REPLACE_MERGE\n - REPLACE_ALL\ndefaultassets:\n - name: query.sql\n value: |\n -- SQL query goes here\n\n Select * from "project.dataset.table";\n \n')),(0,i.kt)("h2",{id:"result"},"Result:"),(0,i.kt)("img",{src:"images/simplify_plugins.png",alt:"Simplify Plugins",width:"800"}),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Executor boot process is standardised and extracted away from plugin developers. Now any arbitrary image can be used for executors."),(0,i.kt)("li",{parentName:"ul"},"At server side, for changes in plugin (dur to plugin release), update the plugin_manager_config and restart the optimus server pod. The plugin manager is expected to reinstall the plugins."),(0,i.kt)("li",{parentName:"ul"},"Client side dependency on plugins is simplified with yaml based plugins.")))}c.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3807],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>g});var a=n(7294);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function l(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):l(l({},t),e)),n},u=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},m="mdxType",c={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,r=e.originalType,s=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),m=p(n),d=i,g=m["".concat(s,".").concat(d)]||m[d]||c[d]||r;return n?a.createElement(g,l(l({ref:t},u),{},{components:n})):a.createElement(g,l({ref:t},u))}));function g(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var r=n.length,l=new Array(r);l[0]=d;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o[m]="string"==typeof e?e:i,l[1]=o;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>c,frontMatter:()=>r,metadata:()=>o,toc:()=>p});var a=n(7462),i=(n(7294),n(3905));const r={},l=void 0,o={unversionedId:"rfcs/simplify_plugin_maintenance",id:"rfcs/simplify_plugin_maintenance",title:"simplify_plugin_maintenance",description:"- Feature Name: Simplify Plugins",source:"@site/docs/rfcs/20220507_simplify_plugin_maintenance.md",sourceDirName:"rfcs",slug:"/rfcs/simplify_plugin_maintenance",permalink:"/optimus/docs/rfcs/simplify_plugin_maintenance",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220507_simplify_plugin_maintenance.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220507,frontMatter:{}},s={},p=[{value:"Background :",id:"background-",level:2},{value:"Changes that trigger a new release in Optimus setup:",id:"changes-that-trigger-a-new-release-in-optimus-setup",level:3},{value:"Release dependencies as per current design",id:"release-dependencies-as-per-current-design",level:3},{value:"1. Avoid Wrapping Executor Images :",id:"1-avoid-wrapping-executor-images--",level:3},{value:"2. Simplify Plugin Installation :",id:"2-simplify-plugin-installation-",level:3},{value:"Approach :",id:"approach-",level:2},{value:"1. Avoid Wrapping Executor Images :",id:"1-avoid-wrapping-executor-images---1",level:3},{value:"2. Simplify Plugin Installations :",id:"2-simplify-plugin-installations-",level:3},{value:"A) Plugin Manager:",id:"a-plugin-manager",level:4},{value:"B) Yaml Plugin Interface: (for client side simplification)",id:"b-yaml-plugin-interface-for-client-side-simplification",level:4},{value:"Result:",id:"result",level:2}],u={toc:p},m="wrapper";function c(e){let{components:t,...n}=e;return(0,i.kt)(m,(0,a.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Feature Name: Simplify Plugins"),(0,i.kt)("li",{parentName:"ul"},"Status: Draft"),(0,i.kt)("li",{parentName:"ul"},"Start Date: 2022-05-07"),(0,i.kt)("li",{parentName:"ul"},"Author: Saikumar")),(0,i.kt)("h1",{id:"summary"},"Summary"),(0,i.kt)("p",null,"The scope of this rfc is to simplify the release and deployment operations w.r.t the optimus plugin ecosystem."),(0,i.kt)("p",null,"The proposal here is to :"),(0,i.kt)("ol",null,(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("strong",{parentName:"li"},"Avoid Wrapping Executor Images")," :",(0,i.kt)("br",{parentName:"li"}),"Decouple the executor_boot_process and the executor as separate containers where the airflow worker launches a pod with init-container (for boot process) adjacent to executor container."),(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("strong",{parentName:"li"},"Simplfy Plugin Installation")," :",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"Server end")," : Install plugins declaratively at runtime instead of manually baking them into the optimus server image (in kubernetes setup)."),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"Client end")," : Plugin interface for client-end is limited to support Version, Survey Questions and Answers etc. that can be extracted out from the current plugin interface and maintained as yaml file which simplifies platform dependent plugin distribution for cli.")))),(0,i.kt)("h1",{id:"technical-design"},"Technical Design"),(0,i.kt)("h2",{id:"background-"},"Background :"),(0,i.kt)("h3",{id:"changes-that-trigger-a-new-release-in-optimus-setup"},"Changes that trigger a new release in Optimus setup:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Executor Image changes"),(0,i.kt)("li",{parentName:"ul"},"Executor Image Wrapper changes"),(0,i.kt)("li",{parentName:"ul"},"Plugin binary changes"),(0,i.kt)("li",{parentName:"ul"},"Optimus binary changes")),(0,i.kt)("h3",{id:"release-dependencies-as-per-current-design"},"Release dependencies as per current design"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"Executor Image release")," -> ",(0,i.kt)("inlineCode",{parentName:"li"},"Executor Wrapper Image release")," -> ",(0,i.kt)("inlineCode",{parentName:"li"},"Plugin binary release")," -> ",(0,i.kt)("inlineCode",{parentName:"li"},"Server release")),(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"Plugin binary release")," -> ",(0,i.kt)("inlineCode",{parentName:"li"},"Server release"))),(0,i.kt)("h3",{id:"1-avoid-wrapping-executor-images--"},"1. ",(0,i.kt)("u",null,"Avoid Wrapping Executor Images")," :"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"The ",(0,i.kt)("inlineCode",{parentName:"li"},"executor_boot_process")," and ",(0,i.kt)("inlineCode",{parentName:"li"},"executor")," are coupled:")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"-- Plugin repo structure\n/task/\n/task/Dockerfile -- task_image\n/task/executor/Dockerfile -- executor_image\n")),(0,i.kt)("p",null,"Executor Wrapper Image (Task Image) :"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"It's a wrapper around the executor_image to facilitate boot mechanism for executor."),(0,i.kt)("li",{parentName:"ul"},"The optimus binary is downloaded during buildtime of this image."),(0,i.kt)("li",{parentName:"ul"},"During runtime, it does as follow :",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Fetch assets, secrets, env from optimus server."),(0,i.kt)("li",{parentName:"ul"},"Load the env and launches the executor process.")))),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre"},"task_image \n | executor_image\n | optimus-bin\n | entrypoint.sh (load assets, env and launch executor)\n")),(0,i.kt)("p",null,"The ",(0,i.kt)("inlineCode",{parentName:"p"},"optimus-bin")," and ",(0,i.kt)("inlineCode",{parentName:"p"},"entrypoint.sh")," are baked into the ",(0,i.kt)("inlineCode",{parentName:"p"},"task_image")," and is being maintained by task/plugin developers."),(0,i.kt)("h3",{id:"2-simplify-plugin-installation-"},"2. ",(0,i.kt)("u",null,"Simplify Plugin Installation")," :"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Plugin binaries are manually installed (baked into optimus image in kubernetes setup). "),(0,i.kt)("li",{parentName:"ul"},"Any change in plugin code demands re-creation of optimus image with new plugin binary, inturn demanding redeployment of optimus server. (in kubernetes setup)"),(0,i.kt)("li",{parentName:"ul"},"At client side, plugin binaries require support for different platforms.")),(0,i.kt)("h2",{id:"approach-"},"Approach :"),(0,i.kt)("h3",{id:"1-avoid-wrapping-executor-images---1"},"1. ",(0,i.kt)("u",null,"Avoid Wrapping Executor Images ")," :"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Decouple the lifecycle of the executor and the boot process as seperate containers/images.")),(0,i.kt)("img",{src:"images/simplify_plugins_executor.png",alt:"Simplify Plugins Executor",width:"800"}),(0,i.kt)("p",null,(0,i.kt)("strong",{parentName:"p"},"Task Boot Sequence"),":"),(0,i.kt)("ol",null,(0,i.kt)("li",{parentName:"ol"},"Airflow worker fetches env and secrets for the job and adds them to the executor pod as environment variables."),(0,i.kt)("li",{parentName:"ol"},"KubernetesPodOperator spawns init-container and executor-container, mounted with shared volume (type emptyDir) for assets."),(0,i.kt)("li",{parentName:"ol"},(0,i.kt)("inlineCode",{parentName:"li"},"init-container")," fetches assets, config, env files and writes onto the shared volume."),(0,i.kt)("li",{parentName:"ol"},"the default entrypoint in the executor-image starts the actual job.")),(0,i.kt)("h3",{id:"2-simplify-plugin-installations-"},"2. ",(0,i.kt)("u",null,"Simplify Plugin Installations")," :"),(0,i.kt)("h4",{id:"a-plugin-manager"},"A) Plugin Manager:"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Currently the plugins are maintained as monorepo and versioned together. For any change in a single plugin, a new tar file containing all plugin binaries is created and released."),(0,i.kt)("li",{parentName:"ul"},"A plugin manager is required to support declarative installation of plugins so that plugins can be independently versioned, packaged and installed."),(0,i.kt)("li",{parentName:"ul"},"This plugin manager consumes a config (plugins_config) and downloads artifacts from a plugin repository.")),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Optimus support for plugin manager as below.",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"optimus plugin install -c config.yaml")," -- at server"))),(0,i.kt)("li",{parentName:"ul"},"Support for different kinds of plugin repositories (like s3, gcs, url, local file system etc..) gives the added flexibility and options to distribute and install the plugin binaries in different ways."),(0,i.kt)("li",{parentName:"ul"},"Plugins are installed at container runtime and this decouples the building of optimus docker image from plugins installations."),(0,i.kt)("li",{parentName:"ul"},"Example for the plugin_config: ")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},"plugin:\n dir: .plugins\n artifacts:\n # local filesystem for dev\n - ../transformers/dist/bq2bq_darwin_arm64/optimus-bq2bq_darwin_arm64\n # any http uri\n - https://github.com/goto/optimus/releases/download/v0.2.5/optimus_0.2.5_linux_arm64.tar.gz\n \n")),(0,i.kt)("h4",{id:"b-yaml-plugin-interface-for-client-side-simplification"},"B) Yaml Plugin Interface: (for client side simplification)"),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Currently plugins are implemented and distributed as binaries and as clients needs to install them, it demands support for different host architectures."),(0,i.kt)("li",{parentName:"ul"},"Since CLI (client side) plugins just require details about plugin such as Version, Suevery Questions etc. the proposal here is to maintain CLI plugins as yaml files."),(0,i.kt)("li",{parentName:"ul"},"Implementation wise, the proposal here is to split the current plugin interface (which only supports interaction with binary plugins) to also accommodate yaml based plugins."),(0,i.kt)("li",{parentName:"ul"},"The above mentioned pluign manager, at server end, would be agnostic about the contents of plugin artifacts from the repository."),(0,i.kt)("li",{parentName:"ul"},"At client side, the CLI could sync the yaml files from the server to stay up-to-date with the server w.r.t plugins."),(0,i.kt)("li",{parentName:"ul"},"At this point, we have the scope to move away from binary plugins except for bq2bq plugin due to its depdendency on ",(0,i.kt)("inlineCode",{parentName:"li"},"ComplileAsset")," and ",(0,i.kt)("inlineCode",{parentName:"li"},"ResolveDependency")," functionalities which are required at server end (not at cli)."),(0,i.kt)("li",{parentName:"ul"},"Handling Bq2Bq plugin:",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},"Move ",(0,i.kt)("inlineCode",{parentName:"li"},"CompileAsset")," functionality as a part of Bq2bq executor."),(0,i.kt)("li",{parentName:"ul"},"Move ",(0,i.kt)("inlineCode",{parentName:"li"},"ResolveDependency")," functionality to optimus core which should support dependecy-resolution on standard-sql"))),(0,i.kt)("li",{parentName:"ul"},"Meanwhile the Bq2bq plugin is handled, the plugin interface can be maintanined in such a way that it also supports binary plugin in addition to yaml (as backward compaitbility feature)."),(0,i.kt)("li",{parentName:"ul"},"The plugin discovery logic should be to load binary if present, else load yaml file; for a single plugin."),(0,i.kt)("li",{parentName:"ul"},"Now that we have yaml files at server, CLI can sync only the yaml files from the server.",(0,i.kt)("ul",{parentName:"li"},(0,i.kt)("li",{parentName:"ul"},(0,i.kt)("inlineCode",{parentName:"li"},"optimus plugin sync -c optimus.yaml"))))),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Example representation of the yaml plugin : ")),(0,i.kt)("pre",null,(0,i.kt)("code",{parentName:"pre",className:"language-yaml"},'name: bq2bq\ndescription: BigQuery to BigQuery transformation task\nplugintype: task\npluginmods:\n - cli\n - dependencyresolver\npluginversion: 0.1.0-SNAPSHOT-27cb56f\napiversion: []\nimage: docker.io/goto/optimus-task-bq2bq-executor:0.1.0-SNAPSHOT-27cb56f\nsecretpath: /tmp/auth.json\ndependson: []\nhooktype: ""\n\nquestions:\n - name: PROJECT\n prompt: Project ID\n help: Destination bigquery project ID\n regexp: ^[a-zA-Z0-9_\\-]+$\n validationerror: invalid name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)\n minlength: 3\n - name: Dataset\n prompt: Dataset Name\n help: Destination bigquery dataset ID\n regexp: ^[a-zA-Z0-9_\\-]+$\n validationerror: invalid name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)\n minlength: 3\n - name: TABLE\n prompt: Table ID\n help: Destination bigquery table ID\n regexp: ^[a-zA-Z0-9_-]+$\n validationerror: invalid table name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)\n minlength: 3\n maxlength: 1024\n - name: LOAD_METHOD\n prompt: Load method to use on destination\n help: |\n APPEND - Append to existing table\n REPLACE - Deletes existing partition and insert result of select query\n MERGE - DML statements, BQ scripts\n REPLACE_MERGE - [Experimental] Advanced replace using merge query\n default: APPEND\n multiselect:\n - APPEND\n - REPLACE\n - MERGE\n - REPLACE_MERGE\n - REPLACE_ALL\ndefaultassets:\n - name: query.sql\n value: |\n -- SQL query goes here\n\n Select * from "project.dataset.table";\n \n')),(0,i.kt)("h2",{id:"result"},"Result:"),(0,i.kt)("img",{src:"images/simplify_plugins.png",alt:"Simplify Plugins",width:"800"}),(0,i.kt)("ul",null,(0,i.kt)("li",{parentName:"ul"},"Executor boot process is standardised and extracted away from plugin developers. Now any arbitrary image can be used for executors."),(0,i.kt)("li",{parentName:"ul"},"At server side, for changes in plugin (dur to plugin release), update the plugin_manager_config and restart the optimus server pod. The plugin manager is expected to reinstall the plugins."),(0,i.kt)("li",{parentName:"ul"},"Client side dependency on plugins is simplified with yaml based plugins.")))}c.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/e7bdcf2c.bcf62a64.js b/assets/js/e7bdcf2c.a0590121.js similarity index 98% rename from assets/js/e7bdcf2c.bcf62a64.js rename to assets/js/e7bdcf2c.a0590121.js index e17c41d871..b57425772d 100644 --- a/assets/js/e7bdcf2c.bcf62a64.js +++ b/assets/js/e7bdcf2c.a0590121.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[7326],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>f});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},l="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,s=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),l=p(n),m=o,f=l["".concat(s,".").concat(m)]||l[m]||d[m]||a;return n?r.createElement(f,c(c({ref:t},u),{},{components:n})):r.createElement(f,c({ref:t},u))}));function f(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,c=new Array(a);c[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i[l]="string"==typeof e?e:o,c[1]=i;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>c,default:()=>d,frontMatter:()=>a,metadata:()=>i,toc:()=>p});var r=n(7462),o=(n(7294),n(3905));const a={},c="Job Run",i={unversionedId:"concepts/job-run",id:"concepts/job-run",title:"Job Run",description:"A job is mainly created to run on schedule. Let\u2019s take a look at what is happening once a scheduler triggers your job to run.",source:"@site/docs/concepts/job-run.md",sourceDirName:"concepts",slug:"/concepts/job-run",permalink:"/optimus/docs/concepts/job-run",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/job-run.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Job",permalink:"/optimus/docs/concepts/job"},next:{title:"Dependency",permalink:"/optimus/docs/concepts/dependency"}},s={},p=[],u={toc:p},l="wrapper";function d(e){let{components:t,...a}=e;return(0,o.kt)(l,(0,r.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"job-run"},"Job Run"),(0,o.kt)("p",null,"A job is mainly created to run on schedule. Let\u2019s take a look at what is happening once a scheduler triggers your job to run. "),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"Job Run Flow",src:n(9473).Z,title:"JobRunFlow",width:"1600",height:"1131"})),(0,o.kt)("p",null,"Note: to test this runtime interactions on your own, take a look and follow the guide on developer environment ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/optimus/tree/main/dev"},"section"),"."))}d.isMDXComponent=!0},9473:(e,t,n)=>{n.d(t,{Z:()=>r});const r=n.p+"assets/images/Concept_JobRun-de8ca16bc3f20f2d6ae96ae1896e3798.png"}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[7326],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>f});var r=n(7294);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function c(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):c(c({},t),e)),n},u=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},l="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,o=e.mdxType,a=e.originalType,s=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),l=p(n),m=o,f=l["".concat(s,".").concat(m)]||l[m]||d[m]||a;return n?r.createElement(f,c(c({ref:t},u),{},{components:n})):r.createElement(f,c({ref:t},u))}));function f(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=n.length,c=new Array(a);c[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i[l]="string"==typeof e?e:o,c[1]=i;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>c,default:()=>d,frontMatter:()=>a,metadata:()=>i,toc:()=>p});var r=n(7462),o=(n(7294),n(3905));const a={},c="Job Run",i={unversionedId:"concepts/job-run",id:"concepts/job-run",title:"Job Run",description:"A job is mainly created to run on schedule. Let\u2019s take a look at what is happening once a scheduler triggers your job to run.",source:"@site/docs/concepts/job-run.md",sourceDirName:"concepts",slug:"/concepts/job-run",permalink:"/optimus/docs/concepts/job-run",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/job-run.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Job",permalink:"/optimus/docs/concepts/job"},next:{title:"Dependency",permalink:"/optimus/docs/concepts/dependency"}},s={},p=[],u={toc:p},l="wrapper";function d(e){let{components:t,...a}=e;return(0,o.kt)(l,(0,r.Z)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"job-run"},"Job Run"),(0,o.kt)("p",null,"A job is mainly created to run on schedule. Let\u2019s take a look at what is happening once a scheduler triggers your job to run. "),(0,o.kt)("p",null,(0,o.kt)("img",{alt:"Job Run Flow",src:n(9473).Z,title:"JobRunFlow",width:"1600",height:"1131"})),(0,o.kt)("p",null,"Note: to test this runtime interactions on your own, take a look and follow the guide on developer environment ",(0,o.kt)("a",{parentName:"p",href:"https://github.com/goto/optimus/tree/main/dev"},"section"),"."))}d.isMDXComponent=!0},9473:(e,t,n)=>{n.d(t,{Z:()=>r});const r=n.p+"assets/images/Concept_JobRun-de8ca16bc3f20f2d6ae96ae1896e3798.png"}}]); \ No newline at end of file diff --git a/assets/js/f00c4084.e225088a.js b/assets/js/f00c4084.397b675f.js similarity index 99% rename from assets/js/f00c4084.e225088a.js rename to assets/js/f00c4084.397b675f.js index d0532e7af4..827002c059 100644 --- a/assets/js/f00c4084.e225088a.js +++ b/assets/js/f00c4084.397b675f.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3921],{3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>b});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function o(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var l=n.createContext({}),c=function(e){var t=n.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):o(o({},t),e)),r},p=function(e){var t=c(e.components);return n.createElement(l.Provider,{value:t},e.children)},s="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,p=u(e,["components","mdxType","originalType","parentName"]),s=c(r),m=a,b=s["".concat(l,".").concat(m)]||s[m]||d[m]||i;return r?n.createElement(b,o(o({ref:t},p),{},{components:r})):n.createElement(b,o({ref:t},p))}));function b(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=r.length,o=new Array(i);o[0]=m;var u={};for(var l in t)hasOwnProperty.call(t,l)&&(u[l]=t[l]);u.originalType=e,u[s]="string"==typeof e?e:a,o[1]=u;for(var c=2;c{r.r(t),r.d(t,{assets:()=>l,contentTitle:()=>o,default:()=>d,frontMatter:()=>i,metadata:()=>u,toc:()=>c});var n=r(7462),a=(r(7294),r(3905));const i={},o="Backup BigQuery Resource",u={unversionedId:"client-guide/backup-bigquery-resource",id:"client-guide/backup-bigquery-resource",title:"Backup BigQuery Resource",description:"Backup is a common prerequisite step to be done before re-running or modifying a resource. Currently, Optimus supports",source:"@site/docs/client-guide/backup-bigquery-resource.md",sourceDirName:"client-guide",slug:"/client-guide/backup-bigquery-resource",permalink:"/optimus/docs/client-guide/backup-bigquery-resource",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/backup-bigquery-resource.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Organizing Specifications",permalink:"/optimus/docs/client-guide/organizing-specifications"},next:{title:"Replay a Job (Backfill)",permalink:"/optimus/docs/client-guide/replay-a-job"}},l={},c=[{value:"Configuring backup details",id:"configuring-backup-details",level:2},{value:"Run a backup",id:"run-a-backup",level:2},{value:"Get the list of backups",id:"get-the-list-of-backups",level:2}],p={toc:c},s="wrapper";function d(e){let{components:t,...r}=e;return(0,a.kt)(s,(0,n.Z)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"backup-bigquery-resource"},"Backup BigQuery Resource"),(0,a.kt)("p",null,"Backup is a common prerequisite step to be done before re-running or modifying a resource. Currently, Optimus supports\nbackup for BigQuery tables and provides dependency resolution, so backup can be also done to all the downstream tables\nas long as it is registered in Optimus and within the same project."),(0,a.kt)("h2",{id:"configuring-backup-details"},"Configuring backup details"),(0,a.kt)("p",null,"Several configurations can be set to have the backup result in your project as your preference. Here are the available\nconfigurations for BigQuery datastore."),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Configuration Key"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"),(0,a.kt)("th",{parentName:"tr",align:null},"Default"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"ttl"),(0,a.kt)("td",{parentName:"tr",align:null},"Time to live in duration"),(0,a.kt)("td",{parentName:"tr",align:null},"720h")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"prefix"),(0,a.kt)("td",{parentName:"tr",align:null},"Prefix of the result table name"),(0,a.kt)("td",{parentName:"tr",align:null},"backup")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"dataset"),(0,a.kt)("td",{parentName:"tr",align:null},"Where the table result should be located"),(0,a.kt)("td",{parentName:"tr",align:null},"optimus_backup")))),(0,a.kt)("p",null,(0,a.kt)("em",{parentName:"p"},"Note: these values can be set in the project configuration.")),(0,a.kt)("h2",{id:"run-a-backup"},"Run a backup"),(0,a.kt)("p",null,"To start a backup, run the following command:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},'$ optimus backup create --resource "resource_name" --project sample-project --namespace sample-namespace\n')),(0,a.kt)("p",null,"After you run the command, prompts will be shown. You will need to answer the questions."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},'$ optimus backup create --resource "resource_name" --project sample-project --namespace sample-namespace\n? Select supported datastore? bigquery\n? Why is this backup needed? backfill due to business logic change\n')),(0,a.kt)("p",null,"Once the backup is finished, the backup results along with where it is located will be shown."),(0,a.kt)("h2",{id:"get-the-list-of-backups"},"Get the list of backups"),(0,a.kt)("p",null,"List of recent backups of a project can be checked using this subcommand:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus backup list --project sample-project\n")),(0,a.kt)("p",null,"Recent backup ID including the resource, when it was created, what is the description or purpose of the backup will\nbe shown. The backup ID is used as a postfix in the backup result name, thus you can find those results in the datastore\n(for example BigQuery) using the backup ID. However, keep in mind that these backup results have an expiry time set."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[3921],{3905:(e,t,r)=>{r.d(t,{Zo:()=>p,kt:()=>b});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function o(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var l=n.createContext({}),c=function(e){var t=n.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):o(o({},t),e)),r},p=function(e){var t=c(e.components);return n.createElement(l.Provider,{value:t},e.children)},s="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,p=u(e,["components","mdxType","originalType","parentName"]),s=c(r),m=a,b=s["".concat(l,".").concat(m)]||s[m]||d[m]||i;return r?n.createElement(b,o(o({ref:t},p),{},{components:r})):n.createElement(b,o({ref:t},p))}));function b(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=r.length,o=new Array(i);o[0]=m;var u={};for(var l in t)hasOwnProperty.call(t,l)&&(u[l]=t[l]);u.originalType=e,u[s]="string"==typeof e?e:a,o[1]=u;for(var c=2;c{r.r(t),r.d(t,{assets:()=>l,contentTitle:()=>o,default:()=>d,frontMatter:()=>i,metadata:()=>u,toc:()=>c});var n=r(7462),a=(r(7294),r(3905));const i={},o="Backup BigQuery Resource",u={unversionedId:"client-guide/backup-bigquery-resource",id:"client-guide/backup-bigquery-resource",title:"Backup BigQuery Resource",description:"Backup is a common prerequisite step to be done before re-running or modifying a resource. Currently, Optimus supports",source:"@site/docs/client-guide/backup-bigquery-resource.md",sourceDirName:"client-guide",slug:"/client-guide/backup-bigquery-resource",permalink:"/optimus/docs/client-guide/backup-bigquery-resource",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/backup-bigquery-resource.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Organizing Specifications",permalink:"/optimus/docs/client-guide/organizing-specifications"},next:{title:"Replay a Job (Backfill)",permalink:"/optimus/docs/client-guide/replay-a-job"}},l={},c=[{value:"Configuring backup details",id:"configuring-backup-details",level:2},{value:"Run a backup",id:"run-a-backup",level:2},{value:"Get the list of backups",id:"get-the-list-of-backups",level:2}],p={toc:c},s="wrapper";function d(e){let{components:t,...r}=e;return(0,a.kt)(s,(0,n.Z)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"backup-bigquery-resource"},"Backup BigQuery Resource"),(0,a.kt)("p",null,"Backup is a common prerequisite step to be done before re-running or modifying a resource. Currently, Optimus supports\nbackup for BigQuery tables and provides dependency resolution, so backup can be also done to all the downstream tables\nas long as it is registered in Optimus and within the same project."),(0,a.kt)("h2",{id:"configuring-backup-details"},"Configuring backup details"),(0,a.kt)("p",null,"Several configurations can be set to have the backup result in your project as your preference. Here are the available\nconfigurations for BigQuery datastore."),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Configuration Key"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"),(0,a.kt)("th",{parentName:"tr",align:null},"Default"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"ttl"),(0,a.kt)("td",{parentName:"tr",align:null},"Time to live in duration"),(0,a.kt)("td",{parentName:"tr",align:null},"720h")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"prefix"),(0,a.kt)("td",{parentName:"tr",align:null},"Prefix of the result table name"),(0,a.kt)("td",{parentName:"tr",align:null},"backup")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"dataset"),(0,a.kt)("td",{parentName:"tr",align:null},"Where the table result should be located"),(0,a.kt)("td",{parentName:"tr",align:null},"optimus_backup")))),(0,a.kt)("p",null,(0,a.kt)("em",{parentName:"p"},"Note: these values can be set in the project configuration.")),(0,a.kt)("h2",{id:"run-a-backup"},"Run a backup"),(0,a.kt)("p",null,"To start a backup, run the following command:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},'$ optimus backup create --resource "resource_name" --project sample-project --namespace sample-namespace\n')),(0,a.kt)("p",null,"After you run the command, prompts will be shown. You will need to answer the questions."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},'$ optimus backup create --resource "resource_name" --project sample-project --namespace sample-namespace\n? Select supported datastore? bigquery\n? Why is this backup needed? backfill due to business logic change\n')),(0,a.kt)("p",null,"Once the backup is finished, the backup results along with where it is located will be shown."),(0,a.kt)("h2",{id:"get-the-list-of-backups"},"Get the list of backups"),(0,a.kt)("p",null,"List of recent backups of a project can be checked using this subcommand:"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-shell"},"$ optimus backup list --project sample-project\n")),(0,a.kt)("p",null,"Recent backup ID including the resource, when it was created, what is the description or purpose of the backup will\nbe shown. The backup ID is used as a postfix in the backup result name, thus you can find those results in the datastore\n(for example BigQuery) using the backup ID. However, keep in mind that these backup results have an expiry time set."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/f1cf558f.4ea152c8.js b/assets/js/f1cf558f.77a12ede.js similarity index 98% rename from assets/js/f1cf558f.4ea152c8.js rename to assets/js/f1cf558f.77a12ede.js index 613da64207..d3a7db338a 100644 --- a/assets/js/f1cf558f.4ea152c8.js +++ b/assets/js/f1cf558f.77a12ede.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6173],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>g});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function o(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=p(n),m=a,g=c["".concat(s,".").concat(m)]||c[m]||d[m]||i;return n?r.createElement(g,o(o({ref:t},u),{},{components:n})):r.createElement(g,o({ref:t},u))}));function g(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,o=new Array(i);o[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[c]="string"==typeof e?e:a,o[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>o,default:()=>d,frontMatter:()=>i,metadata:()=>l,toc:()=>p});var r=n(7462),a=(n(7294),n(3905));const i={},o="Setting up Alert to Job",l={unversionedId:"client-guide/setting-up-alert",id:"client-guide/setting-up-alert",title:"Setting up Alert to Job",description:"There are chances that your job is failing due to some reason or missed the SLA. For these cases, you might want to set",source:"@site/docs/client-guide/setting-up-alert.md",sourceDirName:"client-guide",slug:"/client-guide/setting-up-alert",permalink:"/optimus/docs/client-guide/setting-up-alert",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/setting-up-alert.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Create Job Specifications",permalink:"/optimus/docs/client-guide/create-job-specifications"},next:{title:"Verifying the Jobs",permalink:"/optimus/docs/client-guide/verifying-jobs"}},s={},p=[{value:"Supported Events",id:"supported-events",level:2},{value:"Supported Channels",id:"supported-channels",level:2},{value:"Sample Configuration",id:"sample-configuration",level:2}],u={toc:p},c="wrapper";function d(e){let{components:t,...n}=e;return(0,a.kt)(c,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"setting-up-alert-to-job"},"Setting up Alert to Job"),(0,a.kt)("p",null,"There are chances that your job is failing due to some reason or missed the SLA. For these cases, you might want to set\nthe alerts and get notified as soon as possible."),(0,a.kt)("h2",{id:"supported-events"},"Supported Events"),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Event Type"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"failure"),(0,a.kt)("td",{parentName:"tr",align:null},"Triggered when job run status is failed.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"sla_miss"),(0,a.kt)("td",{parentName:"tr",align:null},"Triggered when the job run does not complete within the duration that you expected. Duration should be specified in the config and should be in string duration.")))),(0,a.kt)("h2",{id:"supported-channels"},"Supported Channels"),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Channel"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Slack"),(0,a.kt)("td",{parentName:"tr",align:null},"Channel/team handle or specific user")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Pagerduty"),(0,a.kt)("td",{parentName:"tr",align:null},"Needing ",(0,a.kt)("inlineCode",{parentName:"td"},"notify_")," secret with pagerduty integration key/routing key")))),(0,a.kt)("h2",{id:"sample-configuration"},"Sample Configuration"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-yaml"},"behavior:\nnotify:\n- 'on': failure/sla_miss\n config :\n duration : 2h45m\n channels:\n - slack://#slack-channel or @team-group or user&gmail.com\n - pagerduty://#pagerduty_service_name\n")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[6173],{3905:(e,t,n)=>{n.d(t,{Zo:()=>u,kt:()=>g});var r=n(7294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function o(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var s=r.createContext({}),p=function(e){var t=r.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},c="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,i=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=p(n),m=a,g=c["".concat(s,".").concat(m)]||c[m]||d[m]||i;return n?r.createElement(g,o(o({ref:t},u),{},{components:n})):r.createElement(g,o({ref:t},u))}));function g(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=n.length,o=new Array(i);o[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l[c]="string"==typeof e?e:a,o[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>o,default:()=>d,frontMatter:()=>i,metadata:()=>l,toc:()=>p});var r=n(7462),a=(n(7294),n(3905));const i={},o="Setting up Alert to Job",l={unversionedId:"client-guide/setting-up-alert",id:"client-guide/setting-up-alert",title:"Setting up Alert to Job",description:"There are chances that your job is failing due to some reason or missed the SLA. For these cases, you might want to set",source:"@site/docs/client-guide/setting-up-alert.md",sourceDirName:"client-guide",slug:"/client-guide/setting-up-alert",permalink:"/optimus/docs/client-guide/setting-up-alert",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/client-guide/setting-up-alert.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Create Job Specifications",permalink:"/optimus/docs/client-guide/create-job-specifications"},next:{title:"Verifying the Jobs",permalink:"/optimus/docs/client-guide/verifying-jobs"}},s={},p=[{value:"Supported Events",id:"supported-events",level:2},{value:"Supported Channels",id:"supported-channels",level:2},{value:"Sample Configuration",id:"sample-configuration",level:2}],u={toc:p},c="wrapper";function d(e){let{components:t,...n}=e;return(0,a.kt)(c,(0,r.Z)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"setting-up-alert-to-job"},"Setting up Alert to Job"),(0,a.kt)("p",null,"There are chances that your job is failing due to some reason or missed the SLA. For these cases, you might want to set\nthe alerts and get notified as soon as possible."),(0,a.kt)("h2",{id:"supported-events"},"Supported Events"),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Event Type"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"failure"),(0,a.kt)("td",{parentName:"tr",align:null},"Triggered when job run status is failed.")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"sla_miss"),(0,a.kt)("td",{parentName:"tr",align:null},"Triggered when the job run does not complete within the duration that you expected. Duration should be specified in the config and should be in string duration.")))),(0,a.kt)("h2",{id:"supported-channels"},"Supported Channels"),(0,a.kt)("table",null,(0,a.kt)("thead",{parentName:"table"},(0,a.kt)("tr",{parentName:"thead"},(0,a.kt)("th",{parentName:"tr",align:null},"Channel"),(0,a.kt)("th",{parentName:"tr",align:null},"Description"))),(0,a.kt)("tbody",{parentName:"table"},(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Slack"),(0,a.kt)("td",{parentName:"tr",align:null},"Channel/team handle or specific user")),(0,a.kt)("tr",{parentName:"tbody"},(0,a.kt)("td",{parentName:"tr",align:null},"Pagerduty"),(0,a.kt)("td",{parentName:"tr",align:null},"Needing ",(0,a.kt)("inlineCode",{parentName:"td"},"notify_")," secret with pagerduty integration key/routing key")))),(0,a.kt)("h2",{id:"sample-configuration"},"Sample Configuration"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-yaml"},"behavior:\nnotify:\n- 'on': failure/sla_miss\n config :\n duration : 2h45m\n channels:\n - slack://#slack-channel or @team-group or user&gmail.com\n - pagerduty://#pagerduty_service_name\n")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/f2458df1.fa91b064.js b/assets/js/f2458df1.8d5e43d6.js similarity index 98% rename from assets/js/f2458df1.fa91b064.js rename to assets/js/f2458df1.8d5e43d6.js index 4cbf08904f..2df67519eb 100644 --- a/assets/js/f2458df1.fa91b064.js +++ b/assets/js/f2458df1.8d5e43d6.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[4230],{3905:(e,t,r)=>{r.d(t,{Zo:()=>s,kt:()=>f});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function l(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var p=n.createContext({}),c=function(e){var t=n.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):l(l({},t),e)),r},s=function(e){var t=c(e.components);return n.createElement(p.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,p=e.parentName,s=i(e,["components","mdxType","originalType","parentName"]),u=c(r),d=a,f=u["".concat(p,".").concat(d)]||u[d]||m[d]||o;return r?n.createElement(f,l(l({ref:t},s),{},{components:r})):n.createElement(f,l({ref:t},s))}));function f(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,l=new Array(o);l[0]=d;var i={};for(var p in t)hasOwnProperty.call(t,p)&&(i[p]=t[p]);i.originalType=e,i[u]="string"==typeof e?e:a,l[1]=i;for(var c=2;c{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>l,default:()=>m,frontMatter:()=>o,metadata:()=>i,toc:()=>c});var n=r(7462),a=(r(7294),r(3905));const o={},l=void 0,i={unversionedId:"rfcs/template",id:"rfcs/template",title:"template",description:"- Feature Name:",source:"@site/docs/rfcs/template.md",sourceDirName:"rfcs",slug:"/rfcs/template",permalink:"/optimus/docs/rfcs/template",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/template.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{}},p={},c=[{value:"Drawbacks",id:"drawbacks",level:2},{value:"Rationale and Alternatives",id:"rationale-and-alternatives",level:2}],s={toc:c},u="wrapper";function m(e){let{components:t,...r}=e;return(0,a.kt)(u,(0,n.Z)({},s,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"Feature Name:"),(0,a.kt)("li",{parentName:"ul"},"Status: draft/in-progress/completed/rejected/obsolete/postponed"),(0,a.kt)("li",{parentName:"ul"},"Start Date: YYYY-MM-DD"),(0,a.kt)("li",{parentName:"ul"},"Authors:")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Remember, you can submit a PR with your RFC before the text is complete.")),(0,a.kt)("h1",{id:"summary"},"Summary"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"What is being proposed"),(0,a.kt)("li",{parentName:"ul"},"Why (short reason)"),(0,a.kt)("li",{parentName:"ul"},"How (short plan)"),(0,a.kt)("li",{parentName:"ul"},"Impact")),(0,a.kt)("p",null,"Audience: all participants"),(0,a.kt)("h1",{id:"technical-design"},"Technical design"),(0,a.kt)("p",null,"Audience: project members, expert users."),(0,a.kt)("h2",{id:"drawbacks"},"Drawbacks"),(0,a.kt)("p",null,"..."),(0,a.kt)("h2",{id:"rationale-and-alternatives"},"Rationale and Alternatives"),(0,a.kt)("p",null,"..."),(0,a.kt)("h1",{id:"unresolved-questions"},"Unresolved questions"),(0,a.kt)("p",null,"Audience: all participants"))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[4230],{3905:(e,t,r)=>{r.d(t,{Zo:()=>s,kt:()=>f});var n=r(7294);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function l(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var p=n.createContext({}),c=function(e){var t=n.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):l(l({},t),e)),r},s=function(e){var t=c(e.components);return n.createElement(p.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,p=e.parentName,s=i(e,["components","mdxType","originalType","parentName"]),u=c(r),d=a,f=u["".concat(p,".").concat(d)]||u[d]||m[d]||o;return r?n.createElement(f,l(l({ref:t},s),{},{components:r})):n.createElement(f,l({ref:t},s))}));function f(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,l=new Array(o);l[0]=d;var i={};for(var p in t)hasOwnProperty.call(t,p)&&(i[p]=t[p]);i.originalType=e,i[u]="string"==typeof e?e:a,l[1]=i;for(var c=2;c{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>l,default:()=>m,frontMatter:()=>o,metadata:()=>i,toc:()=>c});var n=r(7462),a=(r(7294),r(3905));const o={},l=void 0,i={unversionedId:"rfcs/template",id:"rfcs/template",title:"template",description:"- Feature Name:",source:"@site/docs/rfcs/template.md",sourceDirName:"rfcs",slug:"/rfcs/template",permalink:"/optimus/docs/rfcs/template",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/template.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{}},p={},c=[{value:"Drawbacks",id:"drawbacks",level:2},{value:"Rationale and Alternatives",id:"rationale-and-alternatives",level:2}],s={toc:c},u="wrapper";function m(e){let{components:t,...r}=e;return(0,a.kt)(u,(0,n.Z)({},s,r,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"Feature Name:"),(0,a.kt)("li",{parentName:"ul"},"Status: draft/in-progress/completed/rejected/obsolete/postponed"),(0,a.kt)("li",{parentName:"ul"},"Start Date: YYYY-MM-DD"),(0,a.kt)("li",{parentName:"ul"},"Authors:")),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Remember, you can submit a PR with your RFC before the text is complete.")),(0,a.kt)("h1",{id:"summary"},"Summary"),(0,a.kt)("ul",null,(0,a.kt)("li",{parentName:"ul"},"What is being proposed"),(0,a.kt)("li",{parentName:"ul"},"Why (short reason)"),(0,a.kt)("li",{parentName:"ul"},"How (short plan)"),(0,a.kt)("li",{parentName:"ul"},"Impact")),(0,a.kt)("p",null,"Audience: all participants"),(0,a.kt)("h1",{id:"technical-design"},"Technical design"),(0,a.kt)("p",null,"Audience: project members, expert users."),(0,a.kt)("h2",{id:"drawbacks"},"Drawbacks"),(0,a.kt)("p",null,"..."),(0,a.kt)("h2",{id:"rationale-and-alternatives"},"Rationale and Alternatives"),(0,a.kt)("p",null,"..."),(0,a.kt)("h1",{id:"unresolved-questions"},"Unresolved questions"),(0,a.kt)("p",null,"Audience: all participants"))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/f25bba18.79e30085.js b/assets/js/f25bba18.3cf9e21b.js similarity index 99% rename from assets/js/f25bba18.79e30085.js rename to assets/js/f25bba18.3cf9e21b.js index aa16b65935..83367bf068 100644 --- a/assets/js/f25bba18.79e30085.js +++ b/assets/js/f25bba18.3cf9e21b.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[275],{3905:(e,t,a)=>{a.d(t,{Zo:()=>p,kt:()=>h});var i=a(7294);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function n(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,i)}return a}function l(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=i.createContext({}),u=function(e){var t=i.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},p=function(e){var t=u(e.components);return i.createElement(s.Provider,{value:t},e.children)},m="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},c=i.forwardRef((function(e,t){var a=e.components,r=e.mdxType,n=e.originalType,s=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),m=u(a),c=r,h=m["".concat(s,".").concat(c)]||m[c]||d[c]||n;return a?i.createElement(h,l(l({ref:t},p),{},{components:a})):i.createElement(h,l({ref:t},p))}));function h(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var n=a.length,l=new Array(n);l[0]=c;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o[m]="string"==typeof e?e:r,l[1]=o;for(var u=2;u{a.r(t),a.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>d,frontMatter:()=>n,metadata:()=>o,toc:()=>u});var i=a(7462),r=(a(7294),a(3905));const n={},l=void 0,o={unversionedId:"rfcs/optimus_dashboarding",id:"rfcs/optimus_dashboarding",title:"optimus_dashboarding",description:"- Feature Name: Optimus Dashboarding",source:"@site/docs/rfcs/20220517_optimus_dashboarding.md",sourceDirName:"rfcs",slug:"/rfcs/optimus_dashboarding",permalink:"/optimus/docs/rfcs/optimus_dashboarding",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220517_optimus_dashboarding.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220517,frontMatter:{}},s={},u=[{value:"Job Metrics dashboard (aka. Job sumaries page):",id:"job-metrics-dashboard-aka-job-sumaries-page",level:2},{value:"Job lineage view :",id:"job-lineage-view-",level:2},{value:"Approach : Collect info from Airflow events / callbacks :",id:"approach--collect-info-from-airflow-events--callbacks-",level:2},{value:"Optimus Perspective DB model:",id:"optimus-perspective-db-model",level:2},{value:"Compute SLA miss:",id:"compute-sla-miss",level:2},{value:"SLA Definition :",id:"sla-definition-",level:3},{value:"Approach For SLA dashboarding:",id:"approach-for-sla-dashboarding",level:3},{value:"Approach For SLA alerting:",id:"approach-for-sla-alerting",level:3},{value:"Other Considerations:",id:"other-considerations",level:2},{value:"Terminology:",id:"terminology",level:2}],p={toc:u},m="wrapper";function d(e){let{components:t,...a}=e;return(0,r.kt)(m,(0,i.Z)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Feature Name: Optimus Dashboarding"),(0,r.kt)("li",{parentName:"ul"},"Status: Draft"),(0,r.kt)("li",{parentName:"ul"},"Start Date: 2022-05-17"),(0,r.kt)("li",{parentName:"ul"},"Authors: Yash Bhardwaj")),(0,r.kt)("h1",{id:"summary"},"Summary"),(0,r.kt)("p",null,"Optimus users need to frequestly visit airflow webserver which does not provide a optimus perspective of the jobs i.e. is lacks the leniage view and project / namespace level clasification."),(0,r.kt)("p",null,"The proposal here is to capture job related events and state information via airflow event callbacks and job lifecycle events ."),(0,r.kt)("h1",{id:"design"},"Design"),(0,r.kt)("h2",{id:"job-metrics-dashboard-aka-job-sumaries-page"},"Job Metrics dashboard (aka. Job sumaries page):"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Monitor jobs performance stats :",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Duaration per optimus job task/sensor/hook."),(0,r.kt)("li",{parentName:"ul"},"Job completion Sla misses trend."),(0,r.kt)("li",{parentName:"ul"},"Job summaries."))),(0,r.kt)("li",{parentName:"ul"},"Jobs groupings into namespaces and projects.")),(0,r.kt)("hr",null),(0,r.kt)("h2",{id:"job-lineage-view-"},"Job lineage view :"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Currently only columbus provides lineage view."),(0,r.kt)("li",{parentName:"ul"},"That does not reflect the job status and run level dependencies/relationships.")),(0,r.kt)("h2",{id:"approach--collect-info-from-airflow-events--callbacks-"},"Approach : Collect info from Airflow events / callbacks :"),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Get ",(0,r.kt)("inlineCode",{parentName:"p"},"DAG")," lifecycle events from scheduler through:"),(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Airflow triggered Callbacks",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"on_success_callback")," Invoked when the task succeeds"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"on_failure_callback")," Invoked when the task fails"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"sla_miss_callback")," Invoked when a task misses its defined SLA ( SLA here is scheduling delay not to be concused with job completion delay )"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"on_retry_callback")," Invoked when the task is up for retry"))),(0,r.kt)("li",{parentName:"ul"},"Events fired by our Custom logic added into ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Custom operator implimentation "),(0,r.kt)("li",{parentName:"ul"},"airflow job_start_event and job_end_event task"))))),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Information from these events is then relayed to the optimus server. Optimus then writes this into"))),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"OptimusDB")," : for summaries dashboarding and for powering lineage views (in future)")),(0,r.kt)("ol",{start:3},(0,r.kt)("li",{parentName:"ol"},"Reasons for choosing this approach")),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"This is less tightly coupled with the Current chosen Scheduler.",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"If later support is provided for more schedulers, exixting optimus data collection APIs and Optimus Data model can be reused to power our Frontend system.")))),(0,r.kt)("ol",{start:4},(0,r.kt)("li",{parentName:"ol"},"Known limitations:")),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Since this is an event based architecture, collection of Job/DAG state info will be hampered in cases of panic failures for instance DAG python code throwing uncaught exception.",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"even in such cases we will be able to determine the SLA missing jobs ")))),(0,r.kt)("hr",null),(0,r.kt)("h2",{id:"optimus-perspective-db-model"},"Optimus Perspective DB model:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Optimus shall consider each scheduled run and its subsiquent retry as a new job_run "),(0,r.kt)("li",{parentName:"ul"},"sensor/task/hooks run information shall be grouped per job_run ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"To achive this, each sensor/task/hook Run is linked with the job_run.id and job_run.attempt_number"),(0,r.kt)("li",{parentName:"ul"},"while registering a sensor run the latest job_run.attempt for that given schedule time is used to link it."))),(0,r.kt)("li",{parentName:"ul"},"For each job_run(scheduled/re-run) there will only be one row per each sensor/task/hook registered in the Optimus DB.")),(0,r.kt)("hr",null),(0,r.kt)("h2",{id:"compute-sla-miss"},"Compute SLA miss:"),(0,r.kt)("h3",{id:"sla-definition-"},"SLA Definition :"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"if Time duration between the job ",(0,r.kt)("inlineCode",{parentName:"li"},"execution start time")," and ",(0,r.kt)("inlineCode",{parentName:"li"},"job completion time")," excedes the Defined SLA Limit, then that is termed as an SLA breach.")),(0,r.kt)("h3",{id:"approach-for-sla-dashboarding"},"Approach For SLA dashboarding:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Note Job Start Time"),(0,r.kt)("li",{parentName:"ul"},"With Each Job Run, associate the then SLA definition(user defined number in the job.yaml) of the job."),(0,r.kt)("li",{parentName:"ul"},"SLA breach are determined with the read QUERY of Grafana, which works as following ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"SLA breach duration = ",(0,r.kt)("inlineCode",{parentName:"li"},"(min(job_end_time , time.now()) - job_start_time) - SLA_definition")))),(0,r.kt)("li",{parentName:"ul"},"Limitations",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Since this is an event based data collection setup is it possible that a job may have failed/crashed/hangged in such a situation optimus wont get the job finish callback, and hence cant determine the SLA breach ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"To work arround with that, at the time of job run registeration the end time is assumed to be a far future date. In case the job terminates properly we shall be able to determine the correct end_time , otherwise optimus is safe to assume that the job has not finised yet. The job Duration in such case will be the the time since the job has started running.")))))),(0,r.kt)("h3",{id:"approach-for-sla-alerting"},"Approach For SLA alerting:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Ariflow DAG level SLA definiation can be used to get DAG SLA breach event.",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"At the end of each task, a check is conducted to test whether the completed task\u2019s end-time exceeded the SLA OR the start time of the next task exceeded the SLA.")))),(0,r.kt)("hr",null),(0,r.kt)("h2",{id:"other-considerations"},"Other Considerations:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"contrary approached discussed ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Kubernetes Sidecar",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"SideCar lifecycle hooks start/end "),(0,r.kt)("li",{parentName:"ul"},"sideCar to pull details from scheduler/plugin containers and push same to optimus server"))),(0,r.kt)("li",{parentName:"ul"},"Pull Approach",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Callbacks -> statsD to power job sumaries page"),(0,r.kt)("li",{parentName:"ul"},"access airflow API directly from Optimus to power job details view"))))),(0,r.kt)("li",{parentName:"ul"},"Future considerations ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Support for Events fired by Executor :",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"it is expected that even optimus-task and hooks shall independently be able to emit events to optimus notify api. this should help with improved executor observability.")))))),(0,r.kt)("h2",{id:"terminology"},"Terminology:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"Task")," Airflow task operator"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"Job")," Optimus job"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"DAG")," Airflow DAG")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[275],{3905:(e,t,a)=>{a.d(t,{Zo:()=>p,kt:()=>h});var i=a(7294);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function n(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,i)}return a}function l(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var s=i.createContext({}),u=function(e){var t=i.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},p=function(e){var t=u(e.components);return i.createElement(s.Provider,{value:t},e.children)},m="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},c=i.forwardRef((function(e,t){var a=e.components,r=e.mdxType,n=e.originalType,s=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),m=u(a),c=r,h=m["".concat(s,".").concat(c)]||m[c]||d[c]||n;return a?i.createElement(h,l(l({ref:t},p),{},{components:a})):i.createElement(h,l({ref:t},p))}));function h(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var n=a.length,l=new Array(n);l[0]=c;var o={};for(var s in t)hasOwnProperty.call(t,s)&&(o[s]=t[s]);o.originalType=e,o[m]="string"==typeof e?e:r,l[1]=o;for(var u=2;u{a.r(t),a.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>d,frontMatter:()=>n,metadata:()=>o,toc:()=>u});var i=a(7462),r=(a(7294),a(3905));const n={},l=void 0,o={unversionedId:"rfcs/optimus_dashboarding",id:"rfcs/optimus_dashboarding",title:"optimus_dashboarding",description:"- Feature Name: Optimus Dashboarding",source:"@site/docs/rfcs/20220517_optimus_dashboarding.md",sourceDirName:"rfcs",slug:"/rfcs/optimus_dashboarding",permalink:"/optimus/docs/rfcs/optimus_dashboarding",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/rfcs/20220517_optimus_dashboarding.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",sidebarPosition:20220517,frontMatter:{}},s={},u=[{value:"Job Metrics dashboard (aka. Job sumaries page):",id:"job-metrics-dashboard-aka-job-sumaries-page",level:2},{value:"Job lineage view :",id:"job-lineage-view-",level:2},{value:"Approach : Collect info from Airflow events / callbacks :",id:"approach--collect-info-from-airflow-events--callbacks-",level:2},{value:"Optimus Perspective DB model:",id:"optimus-perspective-db-model",level:2},{value:"Compute SLA miss:",id:"compute-sla-miss",level:2},{value:"SLA Definition :",id:"sla-definition-",level:3},{value:"Approach For SLA dashboarding:",id:"approach-for-sla-dashboarding",level:3},{value:"Approach For SLA alerting:",id:"approach-for-sla-alerting",level:3},{value:"Other Considerations:",id:"other-considerations",level:2},{value:"Terminology:",id:"terminology",level:2}],p={toc:u},m="wrapper";function d(e){let{components:t,...a}=e;return(0,r.kt)(m,(0,i.Z)({},p,a,{components:t,mdxType:"MDXLayout"}),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Feature Name: Optimus Dashboarding"),(0,r.kt)("li",{parentName:"ul"},"Status: Draft"),(0,r.kt)("li",{parentName:"ul"},"Start Date: 2022-05-17"),(0,r.kt)("li",{parentName:"ul"},"Authors: Yash Bhardwaj")),(0,r.kt)("h1",{id:"summary"},"Summary"),(0,r.kt)("p",null,"Optimus users need to frequestly visit airflow webserver which does not provide a optimus perspective of the jobs i.e. is lacks the leniage view and project / namespace level clasification."),(0,r.kt)("p",null,"The proposal here is to capture job related events and state information via airflow event callbacks and job lifecycle events ."),(0,r.kt)("h1",{id:"design"},"Design"),(0,r.kt)("h2",{id:"job-metrics-dashboard-aka-job-sumaries-page"},"Job Metrics dashboard (aka. Job sumaries page):"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Monitor jobs performance stats :",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Duaration per optimus job task/sensor/hook."),(0,r.kt)("li",{parentName:"ul"},"Job completion Sla misses trend."),(0,r.kt)("li",{parentName:"ul"},"Job summaries."))),(0,r.kt)("li",{parentName:"ul"},"Jobs groupings into namespaces and projects.")),(0,r.kt)("hr",null),(0,r.kt)("h2",{id:"job-lineage-view-"},"Job lineage view :"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Currently only columbus provides lineage view."),(0,r.kt)("li",{parentName:"ul"},"That does not reflect the job status and run level dependencies/relationships.")),(0,r.kt)("h2",{id:"approach--collect-info-from-airflow-events--callbacks-"},"Approach : Collect info from Airflow events / callbacks :"),(0,r.kt)("ol",null,(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Get ",(0,r.kt)("inlineCode",{parentName:"p"},"DAG")," lifecycle events from scheduler through:"),(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Airflow triggered Callbacks",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"on_success_callback")," Invoked when the task succeeds"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"on_failure_callback")," Invoked when the task fails"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"sla_miss_callback")," Invoked when a task misses its defined SLA ( SLA here is scheduling delay not to be concused with job completion delay )"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"on_retry_callback")," Invoked when the task is up for retry"))),(0,r.kt)("li",{parentName:"ul"},"Events fired by our Custom logic added into ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Custom operator implimentation "),(0,r.kt)("li",{parentName:"ul"},"airflow job_start_event and job_end_event task"))))),(0,r.kt)("li",{parentName:"ol"},(0,r.kt)("p",{parentName:"li"},"Information from these events is then relayed to the optimus server. Optimus then writes this into"))),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"OptimusDB")," : for summaries dashboarding and for powering lineage views (in future)")),(0,r.kt)("ol",{start:3},(0,r.kt)("li",{parentName:"ol"},"Reasons for choosing this approach")),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"This is less tightly coupled with the Current chosen Scheduler.",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"If later support is provided for more schedulers, exixting optimus data collection APIs and Optimus Data model can be reused to power our Frontend system.")))),(0,r.kt)("ol",{start:4},(0,r.kt)("li",{parentName:"ol"},"Known limitations:")),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Since this is an event based architecture, collection of Job/DAG state info will be hampered in cases of panic failures for instance DAG python code throwing uncaught exception.",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"even in such cases we will be able to determine the SLA missing jobs ")))),(0,r.kt)("hr",null),(0,r.kt)("h2",{id:"optimus-perspective-db-model"},"Optimus Perspective DB model:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Optimus shall consider each scheduled run and its subsiquent retry as a new job_run "),(0,r.kt)("li",{parentName:"ul"},"sensor/task/hooks run information shall be grouped per job_run ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"To achive this, each sensor/task/hook Run is linked with the job_run.id and job_run.attempt_number"),(0,r.kt)("li",{parentName:"ul"},"while registering a sensor run the latest job_run.attempt for that given schedule time is used to link it."))),(0,r.kt)("li",{parentName:"ul"},"For each job_run(scheduled/re-run) there will only be one row per each sensor/task/hook registered in the Optimus DB.")),(0,r.kt)("hr",null),(0,r.kt)("h2",{id:"compute-sla-miss"},"Compute SLA miss:"),(0,r.kt)("h3",{id:"sla-definition-"},"SLA Definition :"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"if Time duration between the job ",(0,r.kt)("inlineCode",{parentName:"li"},"execution start time")," and ",(0,r.kt)("inlineCode",{parentName:"li"},"job completion time")," excedes the Defined SLA Limit, then that is termed as an SLA breach.")),(0,r.kt)("h3",{id:"approach-for-sla-dashboarding"},"Approach For SLA dashboarding:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Note Job Start Time"),(0,r.kt)("li",{parentName:"ul"},"With Each Job Run, associate the then SLA definition(user defined number in the job.yaml) of the job."),(0,r.kt)("li",{parentName:"ul"},"SLA breach are determined with the read QUERY of Grafana, which works as following ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"SLA breach duration = ",(0,r.kt)("inlineCode",{parentName:"li"},"(min(job_end_time , time.now()) - job_start_time) - SLA_definition")))),(0,r.kt)("li",{parentName:"ul"},"Limitations",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Since this is an event based data collection setup is it possible that a job may have failed/crashed/hangged in such a situation optimus wont get the job finish callback, and hence cant determine the SLA breach ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"To work arround with that, at the time of job run registeration the end time is assumed to be a far future date. In case the job terminates properly we shall be able to determine the correct end_time , otherwise optimus is safe to assume that the job has not finised yet. The job Duration in such case will be the the time since the job has started running.")))))),(0,r.kt)("h3",{id:"approach-for-sla-alerting"},"Approach For SLA alerting:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"Ariflow DAG level SLA definiation can be used to get DAG SLA breach event.",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"At the end of each task, a check is conducted to test whether the completed task\u2019s end-time exceeded the SLA OR the start time of the next task exceeded the SLA.")))),(0,r.kt)("hr",null),(0,r.kt)("h2",{id:"other-considerations"},"Other Considerations:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},"contrary approached discussed ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Kubernetes Sidecar",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"SideCar lifecycle hooks start/end "),(0,r.kt)("li",{parentName:"ul"},"sideCar to pull details from scheduler/plugin containers and push same to optimus server"))),(0,r.kt)("li",{parentName:"ul"},"Pull Approach",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Callbacks -> statsD to power job sumaries page"),(0,r.kt)("li",{parentName:"ul"},"access airflow API directly from Optimus to power job details view"))))),(0,r.kt)("li",{parentName:"ul"},"Future considerations ",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"Support for Events fired by Executor :",(0,r.kt)("ul",{parentName:"li"},(0,r.kt)("li",{parentName:"ul"},"it is expected that even optimus-task and hooks shall independently be able to emit events to optimus notify api. this should help with improved executor observability.")))))),(0,r.kt)("h2",{id:"terminology"},"Terminology:"),(0,r.kt)("ul",null,(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"Task")," Airflow task operator"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"Job")," Optimus job"),(0,r.kt)("li",{parentName:"ul"},(0,r.kt)("inlineCode",{parentName:"li"},"DAG")," Airflow DAG")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/f3b98b79.7222744e.js b/assets/js/f3b98b79.502ed0da.js similarity index 98% rename from assets/js/f3b98b79.7222744e.js rename to assets/js/f3b98b79.502ed0da.js index 6bee0cd27c..977b7bc455 100644 --- a/assets/js/f3b98b79.7222744e.js +++ b/assets/js/f3b98b79.502ed0da.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8193],{3905:(e,t,r)=>{r.d(t,{Zo:()=>l,kt:()=>f});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function c(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var p=n.createContext({}),i=function(e){var t=n.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):c(c({},t),e)),r},l=function(e){var t=i(e.components);return n.createElement(p.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,a=e.originalType,p=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),u=i(r),d=o,f=u["".concat(p,".").concat(d)]||u[d]||m[d]||a;return r?n.createElement(f,c(c({ref:t},l),{},{components:r})):n.createElement(f,c({ref:t},l))}));function f(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=r.length,c=new Array(a);c[0]=d;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s[u]="string"==typeof e?e:o,c[1]=s;for(var i=2;i{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>c,default:()=>m,frontMatter:()=>a,metadata:()=>s,toc:()=>i});var n=r(7462),o=(r(7294),r(3905));const a={},c="Namespace",s={unversionedId:"concepts/namespace",id:"concepts/namespace",title:"Namespace",description:"A namespace represents a grouping of specified jobs and resources which are accessible only through the namespace owners.",source:"@site/docs/concepts/namespace.md",sourceDirName:"concepts",slug:"/concepts/namespace",permalink:"/optimus/docs/concepts/namespace",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/namespace.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Project",permalink:"/optimus/docs/concepts/project"},next:{title:"Resource",permalink:"/optimus/docs/concepts/resource"}},p={},i=[],l={toc:i},u="wrapper";function m(e){let{components:t,...r}=e;return(0,o.kt)(u,(0,n.Z)({},l,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"namespace"},"Namespace"),(0,o.kt)("p",null,"A namespace represents a grouping of specified jobs and resources which are accessible only through the namespace owners.\nYou may override the project configuration or define the configuration locally at the namespace level. A namespace always\nbelongs to a Project. All Namespaces of a Project share the same infrastructure and the Scheduler."))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[8193],{3905:(e,t,r)=>{r.d(t,{Zo:()=>l,kt:()=>f});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function c(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var p=n.createContext({}),i=function(e){var t=n.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):c(c({},t),e)),r},l=function(e){var t=i(e.components);return n.createElement(p.Provider,{value:t},e.children)},u="mdxType",m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,a=e.originalType,p=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),u=i(r),d=o,f=u["".concat(p,".").concat(d)]||u[d]||m[d]||a;return r?n.createElement(f,c(c({ref:t},l),{},{components:r})):n.createElement(f,c({ref:t},l))}));function f(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var a=r.length,c=new Array(a);c[0]=d;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s[u]="string"==typeof e?e:o,c[1]=s;for(var i=2;i{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>c,default:()=>m,frontMatter:()=>a,metadata:()=>s,toc:()=>i});var n=r(7462),o=(r(7294),r(3905));const a={},c="Namespace",s={unversionedId:"concepts/namespace",id:"concepts/namespace",title:"Namespace",description:"A namespace represents a grouping of specified jobs and resources which are accessible only through the namespace owners.",source:"@site/docs/concepts/namespace.md",sourceDirName:"concepts",slug:"/concepts/namespace",permalink:"/optimus/docs/concepts/namespace",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/namespace.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Project",permalink:"/optimus/docs/concepts/project"},next:{title:"Resource",permalink:"/optimus/docs/concepts/resource"}},p={},i=[],l={toc:i},u="wrapper";function m(e){let{components:t,...r}=e;return(0,o.kt)(u,(0,n.Z)({},l,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"namespace"},"Namespace"),(0,o.kt)("p",null,"A namespace represents a grouping of specified jobs and resources which are accessible only through the namespace owners.\nYou may override the project configuration or define the configuration locally at the namespace level. A namespace always\nbelongs to a Project. All Namespaces of a Project share the same infrastructure and the Scheduler."))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/fa377e30.b52e703f.js b/assets/js/fa377e30.2fba79ef.js similarity index 98% rename from assets/js/fa377e30.b52e703f.js rename to assets/js/fa377e30.2fba79ef.js index 91bf565179..2eadb3a667 100644 --- a/assets/js/fa377e30.b52e703f.js +++ b/assets/js/fa377e30.2fba79ef.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[7181],{3905:(e,t,r)=>{r.d(t,{Zo:()=>u,kt:()=>m});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function c(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function a(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var c=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var p=n.createContext({}),i=function(e){var t=n.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):a(a({},t),e)),r},u=function(e){var t=i(e.components);return n.createElement(p.Provider,{value:t},e.children)},l="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},f=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,c=e.originalType,p=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),l=i(r),f=o,m=l["".concat(p,".").concat(f)]||l[f]||d[f]||c;return r?n.createElement(m,a(a({ref:t},u),{},{components:r})):n.createElement(m,a({ref:t},u))}));function m(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var c=r.length,a=new Array(c);a[0]=f;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s[l]="string"==typeof e?e:o,a[1]=s;for(var i=2;i{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>a,default:()=>d,frontMatter:()=>c,metadata:()=>s,toc:()=>i});var n=r(7462),o=(r(7294),r(3905));const c={},a="Project",s={unversionedId:"concepts/project",id:"concepts/project",title:"Project",description:"A project/tenant represents a group of jobs, resources, and a scheduler with the specified configurations and infrastructure.",source:"@site/docs/concepts/project.md",sourceDirName:"concepts",slug:"/concepts/project",permalink:"/optimus/docs/concepts/project",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/project.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Architecture",permalink:"/optimus/docs/concepts/architecture"},next:{title:"Namespace",permalink:"/optimus/docs/concepts/namespace"}},p={},i=[],u={toc:i},l="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(l,(0,n.Z)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"project"},"Project"),(0,o.kt)("p",null,"A project/tenant represents a group of jobs, resources, and a scheduler with the specified configurations and infrastructure.\nA project contains multiple user-created namespaces, and each has various jobs and resources."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[7181],{3905:(e,t,r)=>{r.d(t,{Zo:()=>u,kt:()=>m});var n=r(7294);function o(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function c(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function a(e){for(var t=1;t=0||(o[r]=e[r]);return o}(e,t);if(Object.getOwnPropertySymbols){var c=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(o[r]=e[r])}return o}var p=n.createContext({}),i=function(e){var t=n.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):a(a({},t),e)),r},u=function(e){var t=i(e.components);return n.createElement(p.Provider,{value:t},e.children)},l="mdxType",d={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},f=n.forwardRef((function(e,t){var r=e.components,o=e.mdxType,c=e.originalType,p=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),l=i(r),f=o,m=l["".concat(p,".").concat(f)]||l[f]||d[f]||c;return r?n.createElement(m,a(a({ref:t},u),{},{components:r})):n.createElement(m,a({ref:t},u))}));function m(e,t){var r=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var c=r.length,a=new Array(c);a[0]=f;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s[l]="string"==typeof e?e:o,a[1]=s;for(var i=2;i{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>a,default:()=>d,frontMatter:()=>c,metadata:()=>s,toc:()=>i});var n=r(7462),o=(r(7294),r(3905));const c={},a="Project",s={unversionedId:"concepts/project",id:"concepts/project",title:"Project",description:"A project/tenant represents a group of jobs, resources, and a scheduler with the specified configurations and infrastructure.",source:"@site/docs/concepts/project.md",sourceDirName:"concepts",slug:"/concepts/project",permalink:"/optimus/docs/concepts/project",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/concepts/project.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"Architecture",permalink:"/optimus/docs/concepts/architecture"},next:{title:"Namespace",permalink:"/optimus/docs/concepts/namespace"}},p={},i=[],u={toc:i},l="wrapper";function d(e){let{components:t,...r}=e;return(0,o.kt)(l,(0,n.Z)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,o.kt)("h1",{id:"project"},"Project"),(0,o.kt)("p",null,"A project/tenant represents a group of jobs, resources, and a scheduler with the specified configurations and infrastructure.\nA project contains multiple user-created namespaces, and each has various jobs and resources."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/fc49bffc.25df574d.js b/assets/js/fc49bffc.51ab710e.js similarity index 99% rename from assets/js/fc49bffc.25df574d.js rename to assets/js/fc49bffc.51ab710e.js index aa17755a80..80396e2a51 100644 --- a/assets/js/fc49bffc.25df574d.js +++ b/assets/js/fc49bffc.51ab710e.js @@ -1 +1 @@ -"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9643],{3905:(t,e,a)=>{a.d(e,{Zo:()=>m,kt:()=>k});var n=a(7294);function r(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function l(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function o(t){for(var e=1;e=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var i=n.createContext({}),u=function(t){var e=n.useContext(i),a=e;return t&&(a="function"==typeof t?t(e):o(o({},e),t)),a},m=function(t){var e=u(t.components);return n.createElement(i.Provider,{value:e},t.children)},s="mdxType",c={inlineCode:"code",wrapper:function(t){var e=t.children;return n.createElement(n.Fragment,{},e)}},d=n.forwardRef((function(t,e){var a=t.components,r=t.mdxType,l=t.originalType,i=t.parentName,m=p(t,["components","mdxType","originalType","parentName"]),s=u(a),d=r,k=s["".concat(i,".").concat(d)]||s[d]||c[d]||l;return a?n.createElement(k,o(o({ref:e},m),{},{components:a})):n.createElement(k,o({ref:e},m))}));function k(t,e){var a=arguments,r=e&&e.mdxType;if("string"==typeof t||r){var l=a.length,o=new Array(l);o[0]=d;var p={};for(var i in e)hasOwnProperty.call(e,i)&&(p[i]=e[i]);p.originalType=t,p[s]="string"==typeof t?t:r,o[1]=p;for(var u=2;u{a.r(e),a.d(e,{assets:()=>i,contentTitle:()=>o,default:()=>c,frontMatter:()=>l,metadata:()=>p,toc:()=>u});var n=a(7462),r=(a(7294),a(3905));const l={},o="Metrics",p={unversionedId:"reference/metrics",id:"reference/metrics",title:"Metrics",description:"Job Change Metrics",source:"@site/docs/reference/metrics.md",sourceDirName:"reference",slug:"/reference/metrics",permalink:"/optimus/docs/reference/metrics",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/reference/metrics.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698812879,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"API",permalink:"/optimus/docs/reference/api"},next:{title:"FAQ",permalink:"/optimus/docs/reference/faq"}},i={},u=[{value:"Job Change Metrics",id:"job-change-metrics",level:2},{value:"JobRun Metrics",id:"jobrun-metrics",level:2},{value:"Resource Metrics",id:"resource-metrics",level:2},{value:"Tenant Metrics",id:"tenant-metrics",level:2},{value:"System Metrics",id:"system-metrics",level:2}],m={toc:u},s="wrapper";function c(t){let{components:e,...a}=t;return(0,r.kt)(s,(0,n.Z)({},m,a,{components:e,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"metrics"},"Metrics"),(0,r.kt)("h2",{id:"job-change-metrics"},"Job Change Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of job changes attempt."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_upload_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of jobs uploaded to scheduler."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_removal_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of jobs removed from scheduler."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_namespace_migrations_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of jobs migrated from a namespace to another."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace_source, namespace_destination")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_replace_all_duration_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Duration of job 'replace-all' process in seconds."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_refresh_duration_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Duration of job 'refresh' process in seconds."),(0,r.kt)("td",{parentName:"tr",align:null},"project")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_validation_duration_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Duration of job 'validation' process in seconds."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace")))),(0,r.kt)("h2",{id:"jobrun-metrics"},"JobRun Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of jobrun events in a job broken by the status, e.g sla_miss, wait_upstream, in_progress, success, failed."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, job, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_sensor_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of sensor run events broken by the event_type, e.g start, retry, success, fail."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, event_type")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_task_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of task run events for a given operator (task name) broken by the event_type, e.g start, retry, success, fail."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, event_type, operator")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_hook_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of hook run events for a given operator (task name) broken by the event_type, e.g start, retry, success, fail."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, event_type, operator")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_replay_requests_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of replay requests for a single job."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, job, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_alerts_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of the alerts triggered broken by the alert type."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, type")))),(0,r.kt)("h2",{id:"resource-metrics"},"Resource Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"resource_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of resource change attempts broken down by the resource type and status."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, datastore, type, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"resource_namespace_migrations_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of resources migrated from a namespace to another namespace."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace_source, namespace_destination")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"resource_upload_all_duration_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"gauge"),(0,r.kt)("td",{parentName:"tr",align:null},"Duration of uploading all resource specification in seconds."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"resource_backup_requests_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of backup requests for a single resource."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, resource, status")))),(0,r.kt)("h2",{id:"tenant-metrics"},"Tenant Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"secret_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of secret change attempts."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, status")))),(0,r.kt)("h2",{id:"system-metrics"},"System Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"application_heartbeat"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Optimus server heartbeat pings."),(0,r.kt)("td",{parentName:"tr",align:null},"-")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"application_uptime_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"gauge"),(0,r.kt)("td",{parentName:"tr",align:null},"Seconds since the application started."),(0,r.kt)("td",{parentName:"tr",align:null},"-")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"notification_queue_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of items queued in the notification channel."),(0,r.kt)("td",{parentName:"tr",align:null},"type")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"notification_worker_batch_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of worker executions in the notification channel."),(0,r.kt)("td",{parentName:"tr",align:null},"type")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"notification_worker_send_err_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of events created and to be sent to writer."),(0,r.kt)("td",{parentName:"tr",align:null},"type")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"publisher_kafka_events_queued_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of events queued to be published to kafka topic."),(0,r.kt)("td",{parentName:"tr",align:null},"-")))))}c.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunkoptimus=self.webpackChunkoptimus||[]).push([[9643],{3905:(t,e,a)=>{a.d(e,{Zo:()=>m,kt:()=>k});var n=a(7294);function r(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function l(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(t);e&&(n=n.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,n)}return a}function o(t){for(var e=1;e=0||(r[a]=t[a]);return r}(t,e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(t);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(r[a]=t[a])}return r}var i=n.createContext({}),u=function(t){var e=n.useContext(i),a=e;return t&&(a="function"==typeof t?t(e):o(o({},e),t)),a},m=function(t){var e=u(t.components);return n.createElement(i.Provider,{value:e},t.children)},s="mdxType",c={inlineCode:"code",wrapper:function(t){var e=t.children;return n.createElement(n.Fragment,{},e)}},d=n.forwardRef((function(t,e){var a=t.components,r=t.mdxType,l=t.originalType,i=t.parentName,m=p(t,["components","mdxType","originalType","parentName"]),s=u(a),d=r,k=s["".concat(i,".").concat(d)]||s[d]||c[d]||l;return a?n.createElement(k,o(o({ref:e},m),{},{components:a})):n.createElement(k,o({ref:e},m))}));function k(t,e){var a=arguments,r=e&&e.mdxType;if("string"==typeof t||r){var l=a.length,o=new Array(l);o[0]=d;var p={};for(var i in e)hasOwnProperty.call(e,i)&&(p[i]=e[i]);p.originalType=t,p[s]="string"==typeof t?t:r,o[1]=p;for(var u=2;u{a.r(e),a.d(e,{assets:()=>i,contentTitle:()=>o,default:()=>c,frontMatter:()=>l,metadata:()=>p,toc:()=>u});var n=a(7462),r=(a(7294),a(3905));const l={},o="Metrics",p={unversionedId:"reference/metrics",id:"reference/metrics",title:"Metrics",description:"Job Change Metrics",source:"@site/docs/reference/metrics.md",sourceDirName:"reference",slug:"/reference/metrics",permalink:"/optimus/docs/reference/metrics",draft:!1,editUrl:"https://github.com/goto/optimus/edit/master/docs/docs/reference/metrics.md",tags:[],version:"current",lastUpdatedBy:"Dery Rahman Ahaddienata",lastUpdatedAt:1698826982,formattedLastUpdatedAt:"Nov 1, 2023",frontMatter:{},sidebar:"docsSidebar",previous:{title:"API",permalink:"/optimus/docs/reference/api"},next:{title:"FAQ",permalink:"/optimus/docs/reference/faq"}},i={},u=[{value:"Job Change Metrics",id:"job-change-metrics",level:2},{value:"JobRun Metrics",id:"jobrun-metrics",level:2},{value:"Resource Metrics",id:"resource-metrics",level:2},{value:"Tenant Metrics",id:"tenant-metrics",level:2},{value:"System Metrics",id:"system-metrics",level:2}],m={toc:u},s="wrapper";function c(t){let{components:e,...a}=t;return(0,r.kt)(s,(0,n.Z)({},m,a,{components:e,mdxType:"MDXLayout"}),(0,r.kt)("h1",{id:"metrics"},"Metrics"),(0,r.kt)("h2",{id:"job-change-metrics"},"Job Change Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of job changes attempt."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_upload_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of jobs uploaded to scheduler."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_removal_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of jobs removed from scheduler."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_namespace_migrations_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of jobs migrated from a namespace to another."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace_source, namespace_destination")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_replace_all_duration_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Duration of job 'replace-all' process in seconds."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_refresh_duration_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Duration of job 'refresh' process in seconds."),(0,r.kt)("td",{parentName:"tr",align:null},"project")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"job_validation_duration_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Duration of job 'validation' process in seconds."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace")))),(0,r.kt)("h2",{id:"jobrun-metrics"},"JobRun Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of jobrun events in a job broken by the status, e.g sla_miss, wait_upstream, in_progress, success, failed."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, job, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_sensor_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of sensor run events broken by the event_type, e.g start, retry, success, fail."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, event_type")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_task_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of task run events for a given operator (task name) broken by the event_type, e.g start, retry, success, fail."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, event_type, operator")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_hook_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of hook run events for a given operator (task name) broken by the event_type, e.g start, retry, success, fail."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, event_type, operator")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_replay_requests_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of replay requests for a single job."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, job, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"jobrun_alerts_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of the alerts triggered broken by the alert type."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, type")))),(0,r.kt)("h2",{id:"resource-metrics"},"Resource Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"resource_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of resource change attempts broken down by the resource type and status."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, datastore, type, status")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"resource_namespace_migrations_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of resources migrated from a namespace to another namespace."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace_source, namespace_destination")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"resource_upload_all_duration_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"gauge"),(0,r.kt)("td",{parentName:"tr",align:null},"Duration of uploading all resource specification in seconds."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"resource_backup_requests_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of backup requests for a single resource."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, resource, status")))),(0,r.kt)("h2",{id:"tenant-metrics"},"Tenant Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"secret_events_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of secret change attempts."),(0,r.kt)("td",{parentName:"tr",align:null},"project, namespace, status")))),(0,r.kt)("h2",{id:"system-metrics"},"System Metrics"),(0,r.kt)("table",null,(0,r.kt)("thead",{parentName:"table"},(0,r.kt)("tr",{parentName:"thead"},(0,r.kt)("th",{parentName:"tr",align:null},"Name"),(0,r.kt)("th",{parentName:"tr",align:null},"Type"),(0,r.kt)("th",{parentName:"tr",align:null},"Description"),(0,r.kt)("th",{parentName:"tr",align:null},"Labels"))),(0,r.kt)("tbody",{parentName:"table"},(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"application_heartbeat"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Optimus server heartbeat pings."),(0,r.kt)("td",{parentName:"tr",align:null},"-")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"application_uptime_seconds"),(0,r.kt)("td",{parentName:"tr",align:null},"gauge"),(0,r.kt)("td",{parentName:"tr",align:null},"Seconds since the application started."),(0,r.kt)("td",{parentName:"tr",align:null},"-")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"notification_queue_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of items queued in the notification channel."),(0,r.kt)("td",{parentName:"tr",align:null},"type")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"notification_worker_batch_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of worker executions in the notification channel."),(0,r.kt)("td",{parentName:"tr",align:null},"type")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"notification_worker_send_err_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of events created and to be sent to writer."),(0,r.kt)("td",{parentName:"tr",align:null},"type")),(0,r.kt)("tr",{parentName:"tbody"},(0,r.kt)("td",{parentName:"tr",align:null},"publisher_kafka_events_queued_total"),(0,r.kt)("td",{parentName:"tr",align:null},"counter"),(0,r.kt)("td",{parentName:"tr",align:null},"Number of events queued to be published to kafka topic."),(0,r.kt)("td",{parentName:"tr",align:null},"-")))))}c.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/runtime~main.af982d20.js b/assets/js/runtime~main.83747779.js similarity index 81% rename from assets/js/runtime~main.af982d20.js rename to assets/js/runtime~main.83747779.js index e10bed44a1..f86eaef1ac 100644 --- a/assets/js/runtime~main.af982d20.js +++ b/assets/js/runtime~main.83747779.js @@ -1 +1 @@ -(()=>{"use strict";var e,f,a,c,d,b={},t={};function r(e){var f=t[e];if(void 0!==f)return f.exports;var a=t[e]={id:e,loaded:!1,exports:{}};return b[e].call(a.exports,a,a.exports,r),a.loaded=!0,a.exports}r.m=b,r.c=t,e=[],r.O=(f,a,c,d)=>{if(!a){var b=1/0;for(i=0;i=d)&&Object.keys(r.O).every((e=>r.O[e](a[o])))?a.splice(o--,1):(t=!1,d0&&e[i-1][2]>d;i--)e[i]=e[i-1];e[i]=[a,c,d]},r.n=e=>{var f=e&&e.__esModule?()=>e.default:()=>e;return r.d(f,{a:f}),f},a=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,r.t=function(e,c){if(1&c&&(e=this(e)),8&c)return e;if("object"==typeof e&&e){if(4&c&&e.__esModule)return e;if(16&c&&"function"==typeof e.then)return e}var d=Object.create(null);r.r(d);var b={};f=f||[null,a({}),a([]),a(a)];for(var t=2&c&&e;"object"==typeof t&&!~f.indexOf(t);t=a(t))Object.getOwnPropertyNames(t).forEach((f=>b[f]=()=>e[f]));return b.default=()=>e,r.d(d,b),d},r.d=(e,f)=>{for(var a in f)r.o(f,a)&&!r.o(e,a)&&Object.defineProperty(e,a,{enumerable:!0,get:f[a]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce(((f,a)=>(r.f[a](e,f),f)),[])),r.u=e=>"assets/js/"+({53:"935f2afb",152:"54f44165",188:"7d18b295",253:"7c550269",275:"f25bba18",528:"b14ed82f",648:"b35a8f38",661:"e8285aa1",732:"cd0afd22",1033:"07653d02",1675:"3278defe",1986:"543eea9a",2019:"303ef138",2106:"12ebdcc3",2147:"cf34028a",2314:"c20e3951",2369:"537b8c88",2389:"174ad3e8",2397:"553314f8",2535:"814f3328",2740:"7e37206e",2743:"2be45fc7",2994:"ab079b58",3089:"a6aa9e1f",3220:"420f7ef9",3390:"6e97b392",3488:"b95ea484",3541:"260cf76e",3608:"9e4087bc",3771:"bf534763",3807:"e22177c2",3921:"f00c4084",4013:"01a85c17",4053:"588bd741",4128:"a09c2993",4195:"c4f5d8e4",4230:"f2458df1",4648:"5ec01bbe",5256:"f5378e77",6070:"81694fd1",6103:"ccc49370",6173:"f1cf558f",6223:"9efad6bf",6301:"6b5e1a0c",6374:"ad92602d",6449:"79c9e2d7",6886:"8a1416ba",6986:"6ae6f4b6",7139:"65608a03",7181:"fa377e30",7326:"e7bdcf2c",7523:"26a45438",7918:"17896441",7997:"d6ec5803",8059:"0b569422",8124:"d4c33874",8193:"f3b98b79",8210:"4f8cfcf0",8462:"d194c8d1",8480:"6d1dc7cf",8610:"6875c492",8728:"d5d1cd4b",8908:"b7fe478b",9047:"7fa9dab1",9364:"2cac66c2",9439:"ae53eb13",9514:"1be78505",9546:"b115b19d",9643:"fc49bffc",9861:"85be924b",9887:"239f3910",9932:"edefa061",9987:"8ce98423"}[e]||e)+"."+{53:"98c39bfe",152:"30190704",188:"ecb7df72",253:"e31700b4",275:"79e30085",528:"a4d8cb0e",648:"0bc983f1",661:"1d3aa5c3",732:"04c3d1c5",1033:"bbc01eb3",1675:"4c2877e3",1986:"15be6dd8",2019:"c3dbc677",2106:"f5034dba",2147:"c343a909",2314:"c0e35d10",2369:"2571e53c",2389:"2cd20389",2397:"d58c9075",2535:"e1fb151b",2740:"d0f2b467",2743:"dff5e95a",2994:"cca0df44",3089:"cc49112f",3220:"825be566",3390:"d9ae6e69",3488:"07276431",3541:"d9d2c58f",3608:"9a153dda",3771:"df52fb30",3807:"9fd87071",3921:"e225088a",4013:"d26e6e00",4053:"eb9469ea",4128:"f0ff2f69",4195:"6859277e",4230:"fa91b064",4648:"75441a54",4972:"2bbd6569",5256:"45289e90",6048:"e31aa954",6070:"0202bb1c",6103:"1fa4d8b2",6173:"4ea152c8",6223:"73405ccb",6301:"971267ff",6374:"f6d05e6f",6449:"b6864922",6886:"a6a96efb",6986:"23b2ede1",7036:"a4663ca1",7139:"ac421215",7181:"b52e703f",7326:"bcf62a64",7523:"bbc0126f",7918:"e850db12",7997:"178dc964",8059:"699f5b7e",8124:"7158cdd1",8193:"7222744e",8210:"0e6f5ff6",8462:"23a61ae6",8480:"f67d68d5",8610:"4563d612",8728:"b6a9aa0c",8908:"8d28011b",9047:"abec0c1c",9364:"b4c030dc",9439:"a144c5b6",9514:"8e99c542",9546:"6e9b75a8",9643:"25df574d",9861:"c47e1e8b",9887:"29ef26a9",9932:"59451ae4",9987:"33350de4"}[e]+".js",r.miniCssF=e=>{},r.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),r.o=(e,f)=>Object.prototype.hasOwnProperty.call(e,f),c={},d="optimus:",r.l=(e,f,a,b)=>{if(c[e])c[e].push(f);else{var t,o;if(void 0!==a)for(var n=document.getElementsByTagName("script"),i=0;i{t.onerror=t.onload=null,clearTimeout(s);var d=c[e];if(delete c[e],t.parentNode&&t.parentNode.removeChild(t),d&&d.forEach((e=>e(a))),f)return f(a)},s=setTimeout(l.bind(null,void 0,{type:"timeout",target:t}),12e4);t.onerror=l.bind(null,t.onerror),t.onload=l.bind(null,t.onload),o&&document.head.appendChild(t)}},r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.p="/optimus/",r.gca=function(e){return e={17896441:"7918","935f2afb":"53","54f44165":"152","7d18b295":"188","7c550269":"253",f25bba18:"275",b14ed82f:"528",b35a8f38:"648",e8285aa1:"661",cd0afd22:"732","07653d02":"1033","3278defe":"1675","543eea9a":"1986","303ef138":"2019","12ebdcc3":"2106",cf34028a:"2147",c20e3951:"2314","537b8c88":"2369","174ad3e8":"2389","553314f8":"2397","814f3328":"2535","7e37206e":"2740","2be45fc7":"2743",ab079b58:"2994",a6aa9e1f:"3089","420f7ef9":"3220","6e97b392":"3390",b95ea484:"3488","260cf76e":"3541","9e4087bc":"3608",bf534763:"3771",e22177c2:"3807",f00c4084:"3921","01a85c17":"4013","588bd741":"4053",a09c2993:"4128",c4f5d8e4:"4195",f2458df1:"4230","5ec01bbe":"4648",f5378e77:"5256","81694fd1":"6070",ccc49370:"6103",f1cf558f:"6173","9efad6bf":"6223","6b5e1a0c":"6301",ad92602d:"6374","79c9e2d7":"6449","8a1416ba":"6886","6ae6f4b6":"6986","65608a03":"7139",fa377e30:"7181",e7bdcf2c:"7326","26a45438":"7523",d6ec5803:"7997","0b569422":"8059",d4c33874:"8124",f3b98b79:"8193","4f8cfcf0":"8210",d194c8d1:"8462","6d1dc7cf":"8480","6875c492":"8610",d5d1cd4b:"8728",b7fe478b:"8908","7fa9dab1":"9047","2cac66c2":"9364",ae53eb13:"9439","1be78505":"9514",b115b19d:"9546",fc49bffc:"9643","85be924b":"9861","239f3910":"9887",edefa061:"9932","8ce98423":"9987"}[e]||e,r.p+r.u(e)},(()=>{var e={1303:0,532:0};r.f.j=(f,a)=>{var c=r.o(e,f)?e[f]:void 0;if(0!==c)if(c)a.push(c[2]);else if(/^(1303|532)$/.test(f))e[f]=0;else{var d=new Promise(((a,d)=>c=e[f]=[a,d]));a.push(c[2]=d);var b=r.p+r.u(f),t=new Error;r.l(b,(a=>{if(r.o(e,f)&&(0!==(c=e[f])&&(e[f]=void 0),c)){var d=a&&("load"===a.type?"missing":a.type),b=a&&a.target&&a.target.src;t.message="Loading chunk "+f+" failed.\n("+d+": "+b+")",t.name="ChunkLoadError",t.type=d,t.request=b,c[1](t)}}),"chunk-"+f,f)}},r.O.j=f=>0===e[f];var f=(f,a)=>{var c,d,b=a[0],t=a[1],o=a[2],n=0;if(b.some((f=>0!==e[f]))){for(c in t)r.o(t,c)&&(r.m[c]=t[c]);if(o)var i=o(r)}for(f&&f(a);n{"use strict";var e,f,a,c,d,b={},t={};function r(e){var f=t[e];if(void 0!==f)return f.exports;var a=t[e]={id:e,loaded:!1,exports:{}};return b[e].call(a.exports,a,a.exports,r),a.loaded=!0,a.exports}r.m=b,r.c=t,e=[],r.O=(f,a,c,d)=>{if(!a){var b=1/0;for(i=0;i=d)&&Object.keys(r.O).every((e=>r.O[e](a[o])))?a.splice(o--,1):(t=!1,d0&&e[i-1][2]>d;i--)e[i]=e[i-1];e[i]=[a,c,d]},r.n=e=>{var f=e&&e.__esModule?()=>e.default:()=>e;return r.d(f,{a:f}),f},a=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,r.t=function(e,c){if(1&c&&(e=this(e)),8&c)return e;if("object"==typeof e&&e){if(4&c&&e.__esModule)return e;if(16&c&&"function"==typeof e.then)return e}var d=Object.create(null);r.r(d);var b={};f=f||[null,a({}),a([]),a(a)];for(var t=2&c&&e;"object"==typeof t&&!~f.indexOf(t);t=a(t))Object.getOwnPropertyNames(t).forEach((f=>b[f]=()=>e[f]));return b.default=()=>e,r.d(d,b),d},r.d=(e,f)=>{for(var a in f)r.o(f,a)&&!r.o(e,a)&&Object.defineProperty(e,a,{enumerable:!0,get:f[a]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce(((f,a)=>(r.f[a](e,f),f)),[])),r.u=e=>"assets/js/"+({53:"935f2afb",152:"54f44165",188:"7d18b295",253:"7c550269",275:"f25bba18",528:"b14ed82f",648:"b35a8f38",661:"e8285aa1",732:"cd0afd22",1033:"07653d02",1675:"3278defe",1986:"543eea9a",2019:"303ef138",2106:"12ebdcc3",2147:"cf34028a",2314:"c20e3951",2369:"537b8c88",2389:"174ad3e8",2397:"553314f8",2535:"814f3328",2740:"7e37206e",2743:"2be45fc7",2994:"ab079b58",3089:"a6aa9e1f",3220:"420f7ef9",3390:"6e97b392",3488:"b95ea484",3541:"260cf76e",3608:"9e4087bc",3771:"bf534763",3807:"e22177c2",3921:"f00c4084",4013:"01a85c17",4053:"588bd741",4128:"a09c2993",4195:"c4f5d8e4",4230:"f2458df1",4648:"5ec01bbe",5256:"f5378e77",6070:"81694fd1",6103:"ccc49370",6173:"f1cf558f",6223:"9efad6bf",6301:"6b5e1a0c",6374:"ad92602d",6449:"79c9e2d7",6886:"8a1416ba",6986:"6ae6f4b6",7139:"65608a03",7181:"fa377e30",7326:"e7bdcf2c",7523:"26a45438",7918:"17896441",7997:"d6ec5803",8059:"0b569422",8124:"d4c33874",8193:"f3b98b79",8210:"4f8cfcf0",8462:"d194c8d1",8480:"6d1dc7cf",8610:"6875c492",8728:"d5d1cd4b",8908:"b7fe478b",9047:"7fa9dab1",9364:"2cac66c2",9439:"ae53eb13",9514:"1be78505",9546:"b115b19d",9643:"fc49bffc",9861:"85be924b",9887:"239f3910",9932:"edefa061",9987:"8ce98423"}[e]||e)+"."+{53:"98c39bfe",152:"0f14f797",188:"ecb7df72",253:"3e410d26",275:"3cf9e21b",528:"4df5e09d",648:"80c646f0",661:"1d3aa5c3",732:"04c3d1c5",1033:"dee0a882",1675:"98e4e90e",1986:"b6af3155",2019:"c3dbc677",2106:"d97869cf",2147:"4ba3eef0",2314:"ace81d80",2369:"e93484a0",2389:"e7d3cc5f",2397:"b1d4357b",2535:"e1fb151b",2740:"d0f2b467",2743:"1fc01053",2994:"658a2062",3089:"cc49112f",3220:"16f54e78",3390:"e912cd31",3488:"bfba3a89",3541:"a8e67b57",3608:"9a153dda",3771:"af003002",3807:"c87118ee",3921:"397b675f",4013:"d26e6e00",4053:"ed5a26d8",4128:"804b23b7",4195:"6859277e",4230:"8d5e43d6",4648:"f7aa917b",4972:"2bbd6569",5256:"45289e90",6048:"e31aa954",6070:"0fa2d851",6103:"1fa4d8b2",6173:"77a12ede",6223:"ba1cc1b9",6301:"971267ff",6374:"436c4a20",6449:"4b681d2a",6886:"eb20bc0d",6986:"15eed989",7036:"a4663ca1",7139:"b6ae1562",7181:"2fba79ef",7326:"a0590121",7523:"bbc0126f",7918:"e850db12",7997:"0fcb8587",8059:"699f5b7e",8124:"1c5e822c",8193:"502ed0da",8210:"40f2e70e",8462:"fef87132",8480:"f67d68d5",8610:"4563d612",8728:"b6a9aa0c",8908:"3beababb",9047:"11b74139",9364:"b4c030dc",9439:"d942e424",9514:"8e99c542",9546:"bcda7022",9643:"51ab710e",9861:"3bd1f4f0",9887:"f3153671",9932:"59451ae4",9987:"3a8598d4"}[e]+".js",r.miniCssF=e=>{},r.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),r.o=(e,f)=>Object.prototype.hasOwnProperty.call(e,f),c={},d="optimus:",r.l=(e,f,a,b)=>{if(c[e])c[e].push(f);else{var t,o;if(void 0!==a)for(var n=document.getElementsByTagName("script"),i=0;i{t.onerror=t.onload=null,clearTimeout(s);var d=c[e];if(delete c[e],t.parentNode&&t.parentNode.removeChild(t),d&&d.forEach((e=>e(a))),f)return f(a)},s=setTimeout(l.bind(null,void 0,{type:"timeout",target:t}),12e4);t.onerror=l.bind(null,t.onerror),t.onload=l.bind(null,t.onload),o&&document.head.appendChild(t)}},r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.p="/optimus/",r.gca=function(e){return e={17896441:"7918","935f2afb":"53","54f44165":"152","7d18b295":"188","7c550269":"253",f25bba18:"275",b14ed82f:"528",b35a8f38:"648",e8285aa1:"661",cd0afd22:"732","07653d02":"1033","3278defe":"1675","543eea9a":"1986","303ef138":"2019","12ebdcc3":"2106",cf34028a:"2147",c20e3951:"2314","537b8c88":"2369","174ad3e8":"2389","553314f8":"2397","814f3328":"2535","7e37206e":"2740","2be45fc7":"2743",ab079b58:"2994",a6aa9e1f:"3089","420f7ef9":"3220","6e97b392":"3390",b95ea484:"3488","260cf76e":"3541","9e4087bc":"3608",bf534763:"3771",e22177c2:"3807",f00c4084:"3921","01a85c17":"4013","588bd741":"4053",a09c2993:"4128",c4f5d8e4:"4195",f2458df1:"4230","5ec01bbe":"4648",f5378e77:"5256","81694fd1":"6070",ccc49370:"6103",f1cf558f:"6173","9efad6bf":"6223","6b5e1a0c":"6301",ad92602d:"6374","79c9e2d7":"6449","8a1416ba":"6886","6ae6f4b6":"6986","65608a03":"7139",fa377e30:"7181",e7bdcf2c:"7326","26a45438":"7523",d6ec5803:"7997","0b569422":"8059",d4c33874:"8124",f3b98b79:"8193","4f8cfcf0":"8210",d194c8d1:"8462","6d1dc7cf":"8480","6875c492":"8610",d5d1cd4b:"8728",b7fe478b:"8908","7fa9dab1":"9047","2cac66c2":"9364",ae53eb13:"9439","1be78505":"9514",b115b19d:"9546",fc49bffc:"9643","85be924b":"9861","239f3910":"9887",edefa061:"9932","8ce98423":"9987"}[e]||e,r.p+r.u(e)},(()=>{var e={1303:0,532:0};r.f.j=(f,a)=>{var c=r.o(e,f)?e[f]:void 0;if(0!==c)if(c)a.push(c[2]);else if(/^(1303|532)$/.test(f))e[f]=0;else{var d=new Promise(((a,d)=>c=e[f]=[a,d]));a.push(c[2]=d);var b=r.p+r.u(f),t=new Error;r.l(b,(a=>{if(r.o(e,f)&&(0!==(c=e[f])&&(e[f]=void 0),c)){var d=a&&("load"===a.type?"missing":a.type),b=a&&a.target&&a.target.src;t.message="Loading chunk "+f+" failed.\n("+d+": "+b+")",t.name="ChunkLoadError",t.type=d,t.request=b,c[1](t)}}),"chunk-"+f,f)}},r.O.j=f=>0===e[f];var f=(f,a)=>{var c,d,b=a[0],t=a[1],o=a[2],n=0;if(b.some((f=>0!==e[f]))){for(c in t)r.o(t,c)&&(r.m[c]=t[c]);if(o)var i=o(r)}for(f&&f(a);n - +
- + \ No newline at end of file diff --git a/blog/index.html b/blog/index.html index 1c7d7ef5f8..5c0093d60f 100644 --- a/blog/index.html +++ b/blog/index.html @@ -10,13 +10,13 @@ - +
- + \ No newline at end of file diff --git a/blog/introducing-optimus/index.html b/blog/introducing-optimus/index.html index 6fdc054308..c05f595248 100644 --- a/blog/introducing-optimus/index.html +++ b/blog/introducing-optimus/index.html @@ -10,13 +10,13 @@ - +
- + \ No newline at end of file diff --git a/blog/tags/goto/index.html b/blog/tags/goto/index.html index d1119d7ca1..cc905ddc6d 100644 --- a/blog/tags/goto/index.html +++ b/blog/tags/goto/index.html @@ -10,13 +10,13 @@ - +

One post tagged with "goto"

View All Tags
- + \ No newline at end of file diff --git a/blog/tags/index.html b/blog/tags/index.html index b4f1d83259..2a3a3392f4 100644 --- a/blog/tags/index.html +++ b/blog/tags/index.html @@ -10,13 +10,13 @@ - +
- + \ No newline at end of file diff --git a/blog/tags/optimus/index.html b/blog/tags/optimus/index.html index ad30ac244d..dc6abb9e0d 100644 --- a/blog/tags/optimus/index.html +++ b/blog/tags/optimus/index.html @@ -10,13 +10,13 @@ - +

One post tagged with "optimus"

View All Tags
- + \ No newline at end of file diff --git a/docs/building-plugin/introduction/index.html b/docs/building-plugin/introduction/index.html index 8f0ce48537..c2df9169f8 100644 --- a/docs/building-plugin/introduction/index.html +++ b/docs/building-plugin/introduction/index.html @@ -10,7 +10,7 @@ - + @@ -30,8 +30,8 @@ is the only interface that is supported in the binary approach to plugins.

Note : Binary plugins augment yaml plugins and they are not standalone.

Plugins can be implemented in any language as long as they can be exported as a single self-contained executable binary and implements a GRPC server. It is recommended to use Go currently for writing plugins because of its cross platform build functionality and to reuse protobuf sdk provided within Optimus core.

Binary Plugins can potentially modify the behavior of Optimus in undesired ways. Exercise caution when adding new -plugins developed by unrecognized developers.

- +plugins developed by unrecognized developers.

+ \ No newline at end of file diff --git a/docs/building-plugin/tutorial/index.html b/docs/building-plugin/tutorial/index.html index 5dffbadc63..a401998857 100644 --- a/docs/building-plugin/tutorial/index.html +++ b/docs/building-plugin/tutorial/index.html @@ -10,7 +10,7 @@ - + @@ -30,8 +30,8 @@ of ways to get them. After extracting API_KEY from secret file, unmarshalling it to json with json.load() send a http request to nasa api. Response can be parsed and printed using the following function:

def print_details(jd):
    """
    Parse and calculate what we need from NASA endpoint response

    :param jd: json data fetched from NASA API
    :return:
    """
    element_count = jd['element_count']
    potentially_hazardous = []
    for search_date in jd['near_earth_objects'].keys():
        for neo in jd['near_earth_objects'][search_date]:
            if neo["is_potentially_hazardous_asteroid"] is True:
                potentially_hazardous.append({
                    "name": neo["name"],
                    "estimated_diameter_km": neo["estimated_diameter"]["kilometers"]["estimated_diameter_max"],
                    "relative_velocity_kmh": neo["close_approach_data"][0]["relative_velocity"]["kilometers_per_hour"]
                })

    print("total tracking: {}\npotential hazardous: {}".format(element_count, len(potentially_hazardous)))
    for haz in potentially_hazardous:
        print("Name: {}\nEstimated Diameter: {} km\nRelative Velocity: {} km/h\n\n".format(
            haz["name"],
            haz["estimated_diameter_km"],
            haz["relative_velocity_kmh"]
        ))
    return

Finish it off by adding the main function

if __name__ == "__main__":
start()

Add requests library in requirements.txt

requests==v2.25.1

Once the python code is ready, wrap this in a Dockerfile

# set base image (host OS)
FROM python:3.8

# set the working directory in the container
RUN mkdir -p /opt
WORKDIR /opt

# copy the content of the local src directory to the working directory
COPY task/neo/executor .

# install dependencies
RUN pip install -r requirements.txt

CMD ["python3", "main.py"]

Creating a yaml plugin​

The Yaml implementation is as simple as providing the plugin details as below.

name: Neo
description: Near earth object tracker
plugintype: task
pluginversion: latest
image: ghcr.io/kushsharma/optimus-task-neo-executor
secretpath: /tmp/.secrets

questions:
- name: RANGE_START
  prompt: Date range start
  help: YYYY-MM-DD format
  required: true
- name: RANGE_END
  prompt: Date range end
  help: YYYY-MM-DD format
  required: true

Based on the usecase, additional validation can be added to the questions section.

This yaml plugin can be placed anywhere, however for this tutorial let’s place it under ../task/neo directory and name it as optimus-plugin-neo.yaml.

Note: As part of this tutorial, we are not providing binary plugin tutorial as it is going to be deprecated.

How to Use​

Before using, let’s install this new plugin in server and client.

Installing the plugin in server​

To use the created plugin in your server, you can simpy add the plugin path in the server config:

plugin:
  artifacts:
   - ../task/neo/optimus-plugin-neo.yaml

To apply the change, you can follow either of these options:

  • Start Optimus server using --install-plugins flag, or
  • Install the plugin separately before starting the server using optimus plugin install command.

Note: Take a look at installing plugins in server guide for more information.

Installing the plugin in client​

Install the plugin in client side by syncing it from server using below command.

$ optimus plugin sync

Once finished, the Neo plugin will be available in the .plugins directory.

Use the plugin in job creation​

Once everything is built and in place, we can generate job specifications that uses neo as the task type.

optimus create job
? What is the job name? is_last_day_on_earth
? Who is the owner of this job? owner@example.io
? Which task to run? neo
? Specify the start date 2022-01-25
? Specify the interval (in crontab notation) 0 2 * * *
? Window truncate to:  d
? Window offset:  0
? Window size:  24h
...
job created successfully is_last_day_on_earth

Create a commit and deploy this specification if you are using optimus with a git managed repositories or send -a REST call or use GRPC, whatever floats your boat.

Once the job has been deployed and run, open the task log and verify something like this

total tracking: 14
potential hazardous: 1
Name: (2014 KP4)
Estimated Diameter: 0.8204270649 km
Relative Velocity: 147052.9914506647 km/h
- +a REST call or use GRPC, whatever floats your boat.

Once the job has been deployed and run, open the task log and verify something like this

total tracking: 14
potential hazardous: 1
Name: (2014 KP4)
Estimated Diameter: 0.8204270649 km
Relative Velocity: 147052.9914506647 km/h
+ \ No newline at end of file diff --git a/docs/client-guide/applying-job-specifications/index.html b/docs/client-guide/applying-job-specifications/index.html index 071756644e..ac1628a2c8 100644 --- a/docs/client-guide/applying-job-specifications/index.html +++ b/docs/client-guide/applying-job-specifications/index.html @@ -10,7 +10,7 @@ - + @@ -21,8 +21,8 @@ needs to refresh all of the jobs in the project from the server, regardless it has changed or not, do run the below command:

$ optimus job refresh --verbose

This refresh command is not taking any specifications as a request. It will only refresh the jobs in the server.

Also, do notice that these replace-all and refresh commands are only for registering the job specifications in the server, including resolving the dependencies. After this, you can compile and upload the jobs to the scheduler using the scheduler upload-all command.

Note: Currently Optimus does not provide a way to deploy only a single job through CLI. This capability is being -supported in the API.

- +supported in the API.

+ \ No newline at end of file diff --git a/docs/client-guide/backup-bigquery-resource/index.html b/docs/client-guide/backup-bigquery-resource/index.html index 2031213715..e07b5f534f 100644 --- a/docs/client-guide/backup-bigquery-resource/index.html +++ b/docs/client-guide/backup-bigquery-resource/index.html @@ -10,7 +10,7 @@ - + @@ -20,8 +20,8 @@ as long as it is registered in Optimus and within the same project.

Configuring backup details​

Several configurations can be set to have the backup result in your project as your preference. Here are the available configurations for BigQuery datastore.

Configuration KeyDescriptionDefault
ttlTime to live in duration720h
prefixPrefix of the result table namebackup
datasetWhere the table result should be locatedoptimus_backup

Note: these values can be set in the project configuration.

Run a backup​

To start a backup, run the following command:

$ optimus backup create --resource "resource_name" --project sample-project --namespace sample-namespace

After you run the command, prompts will be shown. You will need to answer the questions.

$ optimus backup create --resource "resource_name" --project sample-project --namespace sample-namespace
? Select supported datastore? bigquery
? Why is this backup needed? backfill due to business logic change

Once the backup is finished, the backup results along with where it is located will be shown.

Get the list of backups​

List of recent backups of a project can be checked using this subcommand:

$ optimus backup list --project sample-project

Recent backup ID including the resource, when it was created, what is the description or purpose of the backup will be shown. The backup ID is used as a postfix in the backup result name, thus you can find those results in the datastore -(for example BigQuery) using the backup ID. However, keep in mind that these backup results have an expiry time set.

- +(for example BigQuery) using the backup ID. However, keep in mind that these backup results have an expiry time set.

+ \ No newline at end of file diff --git a/docs/client-guide/configuration/index.html b/docs/client-guide/configuration/index.html index 260ad5876b..d7f6d9ffed 100644 --- a/docs/client-guide/configuration/index.html +++ b/docs/client-guide/configuration/index.html @@ -10,7 +10,7 @@ - + @@ -21,8 +21,8 @@ chosen namespaces, including the sub-directories for jobs and resources will be created with the following structure:

sample_project
├── sample_namespace
│   └── jobs
│   └── resources
└── optimus.yaml

Below is the client configuration that has been generated:

version: 1
log:
  level: INFO
  format: ""
host: localhost:9100
project:
  name: sample_project
  config: {}
namespaces:
- name: sample_namespace
  config: {}
  job:
    path: sample_namespace/jobs
  datastore:
    - type: bigquery
      path: sample_namespace/resources
      backup: {}
ConfigurationDescription
VersionSupports only version 1 at the moment.
LogLogging level & format configuration
HostOptimus server host
ProjectChosen Optimus project name and configurations.
NamespacesNamespaces that are owned by the project.

Project​

  • Project name should be unique.
  • Several configs are mandatory for job compilation and deployment use case:
    • storage_path config to store the job compilation result. A path can be anything, for example, a local directory path or a Google Cloud Storage path.
    • scheduler_host being used for job execution and sensors.
    • Specific secrets might be needed for the above configs. Take a look at the detail here.
  • Several configs are optional:
    • scheduler_version to define the scheduler version. More detail is explained here.
  • You can put any other project configurations which can be used in job specifications.

Preset (since v0.10.0)​

Window preset can be configured within the specified project. Preset allows for easier usage of window configuration. For more information, please refer to this page.

Namespaces​

  • Name should be unique in the project.
  • You can put any namespace configurations which can be used in specifications.
  • Job path needs to be properly set so Optimus CLI will able to find all of your job specifications to be processed.
  • For datastore, currently Optimus only accepts bigquery datastore type and you need to set the specification path for this. Also, there is an optional backup config map. Take a look at the backup guide section here -to understand more about this.
- +to understand more about this.
+ \ No newline at end of file diff --git a/docs/client-guide/create-job-specifications/index.html b/docs/client-guide/create-job-specifications/index.html index 7bccdd5069..d42ce0c9dc 100644 --- a/docs/client-guide/create-job-specifications/index.html +++ b/docs/client-guide/create-job-specifications/index.html @@ -10,7 +10,7 @@ - + @@ -37,8 +37,8 @@ concept to know more about it.

For this guide, let’s add a post hook that will audit our BigQuery data using Predator. You can find the Predator plugin YAML file here and have the plugin installed in your server and client.

In order to add a hook to an existing Job, run the following command and answer the corresponding prompts:

$ optimus job addhook
? Please choose the namespace: sample_namespace
? Select a Job sample-project.playground.table1
? Filter expression for extracting transformation rows? __PARTITION__ >= date("{{ .DSTART | Date }}") AND __PARTITION__ < date("{{ .DEND | Date }}")
? Specify the profile/audit result grouping field (empty to not group the result)
? Choose the profiling mode complete

Hook successfully added to sample-project.playground.table1

With the above prompt, we're adding the predator hook post the execution of the primary job. Filter expression -configuration and the rest of the questions are specific to a predator hook, and it might be different for other hooks.

After this, existing job.yaml file will get updated with the new hook config.

hooks:
- name: predator
  config:
    AUDIT_TIME: '{{.EXECUTION_TIME}}'
    BQ_DATASET: '{{.TASK__DATASET}}'
    BQ_PROJECT: '{{.TASK__PROJECT}}'
    BQ_TABLE: '{{.TASK__TABLE}}'
    FILTER: __PARTITION__ >= date("{{ .DSTART | Date }}") AND __PARTITION__ < date("{{ .DEND | Date }}")
    GROUP: ""
    MODE: complete
    PREDATOR_URL: '{{.GLOBAL__PREDATOR_HOST}}'
    SUB_COMMAND: profile_audit
- +configuration and the rest of the questions are specific to a predator hook, and it might be different for other hooks.

After this, existing job.yaml file will get updated with the new hook config.

hooks:
- name: predator
  config:
    AUDIT_TIME: '{{.EXECUTION_TIME}}'
    BQ_DATASET: '{{.TASK__DATASET}}'
    BQ_PROJECT: '{{.TASK__PROJECT}}'
    BQ_TABLE: '{{.TASK__TABLE}}'
    FILTER: __PARTITION__ >= date("{{ .DSTART | Date }}") AND __PARTITION__ < date("{{ .DEND | Date }}")
    GROUP: ""
    MODE: complete
    PREDATOR_URL: '{{.GLOBAL__PREDATOR_HOST}}'
    SUB_COMMAND: profile_audit
+ \ No newline at end of file diff --git a/docs/client-guide/defining-scheduler-version/index.html b/docs/client-guide/defining-scheduler-version/index.html index 3167d6a13c..7b76c66014 100644 --- a/docs/client-guide/defining-scheduler-version/index.html +++ b/docs/client-guide/defining-scheduler-version/index.html @@ -10,7 +10,7 @@ - + @@ -19,8 +19,8 @@ | Version | |---| | 2.1 | -| 2.4 |

- +| 2.4 |

+ \ No newline at end of file diff --git a/docs/client-guide/installing-plugin/index.html b/docs/client-guide/installing-plugin/index.html index bf05304ec2..6a085cc426 100644 --- a/docs/client-guide/installing-plugin/index.html +++ b/docs/client-guide/installing-plugin/index.html @@ -10,15 +10,15 @@ - +

Installing Plugin in Client

Creating job specifications requires a plugin to be installed in the system caller. To simplify the installation, Optimus CLI can sync the YAML plugins supported and served by the Optimus server (with host as declared in project -config) using the following command:

$ optimus plugin sync -c optimus.yaml

Note: This will install plugins in the .plugins folder.

- +config) using the following command:

$ optimus plugin sync -c optimus.yaml

Note: This will install plugins in the .plugins folder.

+ \ No newline at end of file diff --git a/docs/client-guide/manage-bigquery-resource/index.html b/docs/client-guide/manage-bigquery-resource/index.html index 8eaf7ba143..6d3db9fd58 100644 --- a/docs/client-guide/manage-bigquery-resource/index.html +++ b/docs/client-guide/manage-bigquery-resource/index.html @@ -10,15 +10,15 @@ - +

Manage BigQuery Resource

Below is the list of the resource types that Optimus supported:

TypeDescription
datasetResource name format: [project].[dataset]
Spec can includes: table_expiration, description
tableResource name format: [project].[dataset].[table]
Spec can includes: schema, partition, cluster, description
viewResource name format: [project].[dataset].[view]
Spec can includes: view_query, description
external_tableResource name format: [project].[dataset].[table]
Spec can include: schema, source, description

You can create any of the above jobs using the same following format:

$ optimus resource create

Make sure to put the correct resource type as you are intended. Once you fill in the command prompt questions, Optimus will create a file (resource.yaml) in the configured datastore directory. Below is an example of each of the type’s resource specifications.

Dataset​

version: 1
name: sample-project.playground
type: dataset
labels:
  usage: documentation
  owner: optimus
spec:
  description: "example description"
  table_expiration: 24 # in hours

Table​

version: 1
name: sample-project.playground.sample_table
type: table
labels:
  usage: documentation
  owner: optimus
spec:
  description: "example description"
  schema:
  - name: column1
    type: INTEGER
  - name: column2
    type: TIMESTAMP
    description: "example field 2"
    mode: required # (repeated/required/nullable), default: nullable
  - name: column3
    type: STRUCT
    schema: # nested struct schema
  - name: column_a_1
    type: STRING
  cluster:
    using: [column1]
  partition: # leave empty as {} to partition by ingestion time
    field: column2 # column name
    type: day # day/hour, default: day

View​

version: 1
name: sample-project.playground.sample_view
type: view
labels:
  usage: documentation
  owner: optimus
spec:
  description: "example description"
  view_query: |
    Select * from sample-project.playground.sample_table

External Table​

version: 1
name: sample-project.playground.sample_table
type: external_table
labels:
  usage: documentation
  owner: optimus
spec:
  description: "example description"
  schema:
  - name: column1
    type: INTEGER
  - name: column2
    type: TIMESTAMP
    description: "example field 2"
  source:
    type: google_sheets
  uris:
  - https://docs.google.com/spreadsheets/d/spreadsheet_id
  config:
    range: Sheet1!A1:B4 # Range of data to be ingested in the format of [Sheet Name]![Cell Range]
    skip_leading_rows: 1 # Row of records to skip

Upload Resource Specifications​

Once the resource specifications are ready, you can upload all resource specifications using the below command:

$ optimus resource upload-all --verbose

The above command will try to compare the incoming resources to the existing resources in the server. It will create a new resource if it does not exist yet, and modify it if exists, but will not delete any resources. Optimus does not -support BigQuery resource deletion nor the resource record in the Optimus server itself yet.

- +support BigQuery resource deletion nor the resource record in the Optimus server itself yet.

+ \ No newline at end of file diff --git a/docs/client-guide/managing-project-namespace/index.html b/docs/client-guide/managing-project-namespace/index.html index cd1518d9ae..91480dea08 100644 --- a/docs/client-guide/managing-project-namespace/index.html +++ b/docs/client-guide/managing-project-namespace/index.html @@ -10,13 +10,13 @@ - +
-

Managing Project & Namespace

Optimus provides a command to register a new project specified in the client configuration or update if it exists:

$ optimus project register

You are also allowed to register the namespace by using the with-namespaces flag:

$ optimus project register --with-namespaces

You can also check the project configuration that has been registered in your server using:

$ optimus project describe
- +

Managing Project & Namespace

Optimus provides a command to register a new project specified in the client configuration or update if it exists:

$ optimus project register

You are also allowed to register the namespace by using the with-namespaces flag:

$ optimus project register --with-namespaces

You can also check the project configuration that has been registered in your server using:

$ optimus project describe
+ \ No newline at end of file diff --git a/docs/client-guide/managing-secrets/index.html b/docs/client-guide/managing-secrets/index.html index fe57963827..3da9597a81 100644 --- a/docs/client-guide/managing-secrets/index.html +++ b/docs/client-guide/managing-secrets/index.html @@ -10,7 +10,7 @@ - + @@ -20,8 +20,8 @@ Please go through concepts to know more about secrets.

Before we begin, let’s take a look at several mandatory secrets that is used for specific use cases in Optimus.

Secret NameDescription
STORAGETo store compiled jobs if needed.
SCHEDULER_AUTHScheduler credentials. For now, since Optimus only supports Airflow, this will be Airflow [username:password]
BQ_SERVICE_ACCOUNTUsed for any operations involving BigQuery, such as job validation, deployment, run for jobs with BQ to BQ transformation task, as well as for managing BigQuery resources through Optimus.

Registering secret​

Register a secret by running the following command:

$ optimus secret set someSecret someSecretValue

By default, Optimus will encode the secret value. However, to register a secret that has been encoded, run the following command instead:

$ optimus secret set someSecret encodedSecretValue --base64

There is also a flexibility to register using an existing secret file, instead of providing the secret value in the command.

$ optimus secret set someSecret --file=/path/to/secret

Secret can also be set to a specific namespace which can only be used by the jobs/resources in the namespace. To register, run the following command:

$ optimus secret set someSecret someSecretValue --namespace someNamespace

Please note that registering a secret that already exists will result in an error. Modifying an existing secret -can be done using the Update command.

Updating a secret​

The update-only flag is generally used when you explicitly only want to update a secret that already exists and doesn't want to create it by mistake.

$ optimus secret set someSecret someSecretValue --update-only

It will return an error if the secret to update does not exist already.

Listing secrets​

The list command can be used to show the user-defined secrets which are registered with Optimus. It will list the namespace associated with a secret.

$ optimus secret list
Secrets for project: optimus-local
NAME    |                    DIGEST                    | NAMESPACE |         DATE
-------------+----------------------------------------------+-----------+----------------------
secret1   | SIBzsgUuHnExBY4qSzqcrlrb+3zCAHGu/4Fv1O8eMI8= |     *     | 2022-04-12T04:30:45Z

It shows a digest for the encrypted secret, so as not to send the cleartext password on the network.

- +can be done using the Update command.

Updating a secret​

The update-only flag is generally used when you explicitly only want to update a secret that already exists and doesn't want to create it by mistake.

$ optimus secret set someSecret someSecretValue --update-only

It will return an error if the secret to update does not exist already.

Listing secrets​

The list command can be used to show the user-defined secrets which are registered with Optimus. It will list the namespace associated with a secret.

$ optimus secret list
Secrets for project: optimus-local
NAME    |                    DIGEST                    | NAMESPACE |         DATE
-------------+----------------------------------------------+-----------+----------------------
secret1   | SIBzsgUuHnExBY4qSzqcrlrb+3zCAHGu/4Fv1O8eMI8= |     *     | 2022-04-12T04:30:45Z

It shows a digest for the encrypted secret, so as not to send the cleartext password on the network.

+ \ No newline at end of file diff --git a/docs/client-guide/organizing-specifications/index.html b/docs/client-guide/organizing-specifications/index.html index fcf9aa593f..737e0df9f0 100644 --- a/docs/client-guide/organizing-specifications/index.html +++ b/docs/client-guide/organizing-specifications/index.html @@ -10,7 +10,7 @@ - + @@ -20,8 +20,8 @@ assuming there are 2 namespaces in a project.

.
├── optimus.yaml
├── preset.yaml
├── README.md
├── namespace-1
│   ├── jobs
|   │   ├── job1
|   │   ├── job2
|   │   └── this.yaml
│   └── resources
|       ├── bigquery
│       │   ├── table1
│       │   ├── table2
|       |   └── this.yaml
│       └── postgres
│           └── table1
├── namespace-2
├── jobs
└── resources

You might have also noticed there are this.yaml files being used in some directories. This file is used to share a single set of configurations across multiple sub-directories. For example, if you create a file at /namespace-1/jobs/this.yaml, then all subdirectories inside /namespaces-1/jobs will inherit this config as defaults. -If the same config is specified in subdirectory, then subdirectory will override the parent defaults.

For example a this.yaml in /namespace-1/jobs

version: 1
schedule:
  interval: @daily
task:
  name: bq2bq
  config:
    BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"
behavior:
  depends_on_past: false
  retry:
    count: 1
    delay: 5s

and a job.yaml in /namespace-1/jobs/job1

name: sample_replace
owner: optimus@example.io
schedule:
  start_date: "2020-09-25"
  interval: 0 10 * * *
behavior:
  depends_on_past: true
task:
  name: bq2bq
  config:
    project: project_name
    dataset: project_dataset
    table: sample_replace
    load_method: REPLACE
  window:
    size: 48h
    offset: 24h

will result in final computed job.yaml during deployment as

version: 1
name: sample_replace
owner: optimus@example.io
schedule:
  start_date: "2020-10-06"
  interval: 0 10 * * *
behavior:
  depends_on_past: true
  retry:
    count: 1
    delay: 5s
task:
  name: bq2bq
  config:
    project: project_name
    dataset: project_dataset
    table: sample_replace
    load_method: REPLACE
    BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"
  window:
    size: 48h
    offset: 24h

Preset (since v0.10.0)

Window preset allows for easier usage of window configuration and can be specified through a YAML file. It is optional in nature and is recommended to be put in the same directory as optimus.yaml if being set. For more information on how to utilize window preset, please check this page.

- +If the same config is specified in subdirectory, then subdirectory will override the parent defaults.

For example a this.yaml in /namespace-1/jobs

version: 1
schedule:
  interval: @daily
task:
  name: bq2bq
  config:
    BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"
behavior:
  depends_on_past: false
  retry:
    count: 1
    delay: 5s

and a job.yaml in /namespace-1/jobs/job1

name: sample_replace
owner: optimus@example.io
schedule:
  start_date: "2020-09-25"
  interval: 0 10 * * *
behavior:
  depends_on_past: true
task:
  name: bq2bq
  config:
    project: project_name
    dataset: project_dataset
    table: sample_replace
    load_method: REPLACE
  window:
    size: 48h
    offset: 24h

will result in final computed job.yaml during deployment as

version: 1
name: sample_replace
owner: optimus@example.io
schedule:
  start_date: "2020-10-06"
  interval: 0 10 * * *
behavior:
  depends_on_past: true
  retry:
    count: 1
    delay: 5s
task:
  name: bq2bq
  config:
    project: project_name
    dataset: project_dataset
    table: sample_replace
    load_method: REPLACE
    BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"
  window:
    size: 48h
    offset: 24h

Preset (since v0.10.0)

Window preset allows for easier usage of window configuration and can be specified through a YAML file. It is optional in nature and is recommended to be put in the same directory as optimus.yaml if being set. For more information on how to utilize window preset, please check this page.

+ \ No newline at end of file diff --git a/docs/client-guide/replay-a-job/index.html b/docs/client-guide/replay-a-job/index.html index 5f83162c2c..15b34250e5 100644 --- a/docs/client-guide/replay-a-job/index.html +++ b/docs/client-guide/replay-a-job/index.html @@ -10,7 +10,7 @@ - + @@ -19,8 +19,8 @@ various reasons. Optimus provides a way to do this using Replay. Please go through concepts to know more about it.

Run a replay​

To run a replay, run the following command:

$ optimus replay create {job_name} {start_time} {end_time} [flags]

Example:

$ optimus replay create sample-job 2023-03-01T00:00:00Z 2023-03-02T15:00:00Z --parallel --project sample-project --namespace-name sample-namespace

Replay accepts three arguments, first is the DAG name that is used in Optimus specification, second is the scheduled start time of replay, and third is the scheduled end time (optional) of replay.

Once your request has been successfully replayed, this means that Replay has cleared the requested runs in the scheduler. Please wait until the scheduler finishes scheduling and running those tasks.

Get a replay status​

You can check the replay status using the replay ID given previously and use in this command:

$ optimus replay status {replay_id} [flag]

You will see the latest replay status including the status of each run of your replay.

Get list of replays​

List of recent replay of a project can be checked using this sub command:

$ optimus replay list [flag]

Recent replay ID including the job, time window, replay time, and status will be shown. To check the detailed status -of a replay, please use the status sub command.

- +of a replay, please use the status sub command.

+ \ No newline at end of file diff --git a/docs/client-guide/setting-up-alert/index.html b/docs/client-guide/setting-up-alert/index.html index 3013dfa2e7..bb559bba46 100644 --- a/docs/client-guide/setting-up-alert/index.html +++ b/docs/client-guide/setting-up-alert/index.html @@ -10,14 +10,14 @@ - +

Setting up Alert to Job

There are chances that your job is failing due to some reason or missed the SLA. For these cases, you might want to set -the alerts and get notified as soon as possible.

Supported Events​

Event TypeDescription
failureTriggered when job run status is failed.
sla_missTriggered when the job run does not complete within the duration that you expected. Duration should be specified in the config and should be in string duration.

Supported Channels​

ChannelDescription
SlackChannel/team handle or specific user
PagerdutyNeeding notify_<pagerduty_service_name> secret with pagerduty integration key/routing key

Sample Configuration​

behavior:
notify:
- 'on': failure/sla_miss
  config :
    duration : 2h45m
  channels:
    - slack://#slack-channel or @team-group or user&gmail.com
    - pagerduty://#pagerduty_service_name
- +the alerts and get notified as soon as possible.

Supported Events​

Event TypeDescription
failureTriggered when job run status is failed.
sla_missTriggered when the job run does not complete within the duration that you expected. Duration should be specified in the config and should be in string duration.

Supported Channels​

ChannelDescription
SlackChannel/team handle or specific user
PagerdutyNeeding notify_<pagerduty_service_name> secret with pagerduty integration key/routing key

Sample Configuration​

behavior:
notify:
- 'on': failure/sla_miss
  config :
    duration : 2h45m
  channels:
    - slack://#slack-channel or @team-group or user&gmail.com
    - pagerduty://#pagerduty_service_name
+ \ No newline at end of file diff --git a/docs/client-guide/uploading-jobs-to-scheduler/index.html b/docs/client-guide/uploading-jobs-to-scheduler/index.html index 401eacff31..efe904e5a4 100644 --- a/docs/client-guide/uploading-jobs-to-scheduler/index.html +++ b/docs/client-guide/uploading-jobs-to-scheduler/index.html @@ -10,15 +10,15 @@ - +

Uploading Job to Scheduler

Compile and upload all jobs in the project by using this command:

$ optimus scheduler upload-all

Note: add --config flag if you are not in the same directory with your client configuration (optimus.yaml).

This command will compile all of the jobs in the project to Airflow DAG files and will store the result to the path that has been set as STORAGE_PATH in the project configuration. Do note that STORAGE secret might be needed if -the storage requires a credential.

Once you have the DAG files in the storage, you can sync the files to Airflow as you’d like.

- +the storage requires a credential.

Once you have the DAG files in the storage, you can sync the files to Airflow as you’d like.

+ \ No newline at end of file diff --git a/docs/client-guide/verifying-jobs/index.html b/docs/client-guide/verifying-jobs/index.html index cb03b938cf..81b1e702d3 100644 --- a/docs/client-guide/verifying-jobs/index.html +++ b/docs/client-guide/verifying-jobs/index.html @@ -10,7 +10,7 @@ - + @@ -23,8 +23,8 @@ to your Optimus server.

To inspect a job in your local:

$ optimus job inspect <job_name>

To inspect a job in the server:

$ optimus job inspect <job_name> --server

You will find mainly 3 sections for inspection:

  • Basic Info: Optimus will print the job’s specification, including the resource destination & sources (if any), and whether it has any soft warning.
  • Upstreams: Will prints what are the jobs that this job depends on. Do notice there might be internal upstreams, external (cross-server) upstreams, HTTP upstreams, and unknown upstreams (not registered in Optimus).
  • Downstreams: -Will prints what are the jobs that depends on this job.
- +Will prints what are the jobs that depends on this job.
+ \ No newline at end of file diff --git a/docs/client-guide/work-with-extension/index.html b/docs/client-guide/work-with-extension/index.html index fc84a92146..c6065e36b8 100644 --- a/docs/client-guide/work-with-extension/index.html +++ b/docs/client-guide/work-with-extension/index.html @@ -10,7 +10,7 @@ - + @@ -33,8 +33,8 @@ being too long or the user just wants to change it.

Example:

$ optimus extension valor rename vl

Uninstall​

Uninstalls extension as a whole or only a specific tag. This allows the user to do some cleanup to preserve some storage or to resolve some issues. By default, Optimus will uninstall the extension as a whole. To target a specific tag, use the flag --tag.

Example:

$ optimus extension valor uninstall

Upgrade​

Upgrade allows the user to upgrade a certain extension to its latest tag. Although the user can use the install command, -using this command is shorter and easier as the user only needs to specify the installed extension.

Example:

$ optimus extension valor upgrade
- +using this command is shorter and easier as the user only needs to specify the installed extension.

Example:

$ optimus extension valor upgrade
+ \ No newline at end of file diff --git a/docs/concepts/architecture/index.html b/docs/concepts/architecture/index.html index 4af96180f5..281d7349d5 100644 --- a/docs/concepts/architecture/index.html +++ b/docs/concepts/architecture/index.html @@ -10,7 +10,7 @@ - + @@ -23,8 +23,8 @@ capability is extended through plugins and users can customize based on their needs on what plugins to use. Plugins can be defined through a yaml specification. At the time of execution whatever the image that is configured in the plugin image will be executed.

Scheduler (Airflow)​

Scheduler is responsible for scheduling all the user defined jobs. Currently, optimus supports only Airflow as -the scheduler, support for more schedulers can be added.

- +the scheduler, support for more schedulers can be added.

+ \ No newline at end of file diff --git a/docs/concepts/dependency/index.html b/docs/concepts/dependency/index.html index ab6bac434e..05ded6dc49 100644 --- a/docs/concepts/dependency/index.html +++ b/docs/concepts/dependency/index.html @@ -10,7 +10,7 @@ - + @@ -19,8 +19,8 @@ non-managed like an S3 bucket. If the dependency is managed by Optimus, it is obvious that in an ETL pipeline, it is required for the dependency to finish successfully first before the dependent job can start.

There are 2 types of dependency depending on how to configure it:

TypeDescription
InferredAutomatically detected through assets. The logic on how to detect the dependency is configured in each of the plugins.
StaticConfigured through job.yaml

Optimus also supports job dependency to cross-optimus servers. These Optimus servers are considered external resource managers, where Optimus will look for the job sources that have not been resolved internally and create the dependency. -These resource managers should be configured in the server configuration.

- +These resource managers should be configured in the server configuration.

+ \ No newline at end of file diff --git a/docs/concepts/intervals-and-windows/index.html b/docs/concepts/intervals-and-windows/index.html index a7876c0dde..88b345bbe3 100644 --- a/docs/concepts/intervals-and-windows/index.html +++ b/docs/concepts/intervals-and-windows/index.html @@ -10,7 +10,7 @@ - + @@ -41,8 +41,8 @@ The following is an example to refer the preset file under project configuration:

version: 1
log:
  ...
host: localhost:9100
project:
  name: development_project
  preset_path: ./preset.yaml # points to preset file
  config:
    ...
namespaces:
  ...

In the above example, a new field is present, named preset_path. This path refers to where the preset file is located.

  • Preset Reference for Job Specification

Now, if the other two components are met, where the window preset file is specified and this file is referenced by the project, it means it is ready to be used. And the way to use it is by referencing which preset to be used in whichever job requires it. The following is an example of its usage:

version: 1 # preset always use window version 2
name: sample-project.playground.table1
owner: sample_owner
schedule:
  ...
behavior:
  ...
task:
  name: bq2bq
  config:
    ...
  window:
    preset: yesterday
labels:
  ...
hooks: []
dependencies: []

Important note, preset is optional in nature. It means that even if the preset is specified, the user can still use -the custom window configuration depending on their need.

- +the custom window configuration depending on their need.

+ \ No newline at end of file diff --git a/docs/concepts/job-run/index.html b/docs/concepts/job-run/index.html index f7b5d0aff6..d7a07ea5b6 100644 --- a/docs/concepts/job-run/index.html +++ b/docs/concepts/job-run/index.html @@ -10,13 +10,13 @@ - +
-

Job Run

A job is mainly created to run on schedule. Let’s take a look at what is happening once a scheduler triggers your job to run.

Job Run Flow

Note: to test this runtime interactions on your own, take a look and follow the guide on developer environment section.

- +

Job Run

A job is mainly created to run on schedule. Let’s take a look at what is happening once a scheduler triggers your job to run.

Job Run Flow

Note: to test this runtime interactions on your own, take a look and follow the guide on developer environment section.

+ \ No newline at end of file diff --git a/docs/concepts/job/index.html b/docs/concepts/job/index.html index 2a5e6247db..da10ffac5f 100644 --- a/docs/concepts/job/index.html +++ b/docs/concepts/job/index.html @@ -10,7 +10,7 @@ - + @@ -25,8 +25,8 @@ from the base transformation or from a global configuration store.

The fundamental difference between a hook and a task is, a task can have dependencies over other jobs inside the repository whereas a hook can only depend on other hooks within the job.

Asset​

There could be an asset folder along with the job.yaml file generated via optimus when a new job is created. This is a shared folder across base transformation task and all associated hooks. Assets can use macros and functions powered by -Go templating engine.

Section of code can be imported from different asset files using template. For example:

  • File partials.gtpl
DECLARE t1 TIMESTAMP;
  • Another file query.sql
{{template "partials.gtpl"}}
SET t1 = '2021-02-10T10:00:00+00:00';

During execution query.sql will be rendered as:

DECLARE t1 TIMESTAMP;
SET t1 = '2021-02-10T10:00:00+00:00';

whereas partials.gtpl will be left as it is because file was saved with .gtpl extension.

Similarly, a single file can contain multiple blocks of code that can function as macro of code replacement. For example:

  • file.data
Name: {{ template "name"}}, Gender: {{ template "gender" }}
  • partials.gtpl
{{- define "name" -}} Adam {{- end}}
{{- define "gender" -}} Male {{- end}}

This will render file.data as

Name: Adam, Gender: Male
- +Go templating engine.

Section of code can be imported from different asset files using template. For example:

  • File partials.gtpl
DECLARE t1 TIMESTAMP;
  • Another file query.sql
{{template "partials.gtpl"}}
SET t1 = '2021-02-10T10:00:00+00:00';

During execution query.sql will be rendered as:

DECLARE t1 TIMESTAMP;
SET t1 = '2021-02-10T10:00:00+00:00';

whereas partials.gtpl will be left as it is because file was saved with .gtpl extension.

Similarly, a single file can contain multiple blocks of code that can function as macro of code replacement. For example:

  • file.data
Name: {{ template "name"}}, Gender: {{ template "gender" }}
  • partials.gtpl
{{- define "name" -}} Adam {{- end}}
{{- define "gender" -}} Male {{- end}}

This will render file.data as

Name: Adam, Gender: Male
+ \ No newline at end of file diff --git a/docs/concepts/macros/index.html b/docs/concepts/macros/index.html index 4923167822..28ca12e3d9 100644 --- a/docs/concepts/macros/index.html +++ b/docs/concepts/macros/index.html @@ -10,14 +10,14 @@ - +

Macros

Macros are special variables that will be replaced by actual values when before execution. Custom macros are not -supported yet.

Below listed the in built macros supported in optimus.

MacrosDescription
{{.DSTART}}start date/datetime of the window as 2021-02-10T10:00:00+00:00 that is, RFC3339
{{.DEND}}end date/datetime of the window, as RFC3339
{{.JOB_DESTINATION}}full qualified table name used in DML statement
{{.EXECUTION_TIME}}timestamp when the specific job run starts

Take a detailed look at the windows concept and example here.

- +supported yet.

Below listed the in built macros supported in optimus.

MacrosDescription
{{.DSTART}}start date/datetime of the window as 2021-02-10T10:00:00+00:00 that is, RFC3339
{{.DEND}}end date/datetime of the window, as RFC3339
{{.JOB_DESTINATION}}full qualified table name used in DML statement
{{.EXECUTION_TIME}}timestamp when the specific job run starts

Take a detailed look at the windows concept and example here.

+ \ No newline at end of file diff --git a/docs/concepts/namespace/index.html b/docs/concepts/namespace/index.html index 76dc72fbbd..e4004f958e 100644 --- a/docs/concepts/namespace/index.html +++ b/docs/concepts/namespace/index.html @@ -10,15 +10,15 @@ - +

Namespace

A namespace represents a grouping of specified jobs and resources which are accessible only through the namespace owners. You may override the project configuration or define the configuration locally at the namespace level. A namespace always -belongs to a Project. All Namespaces of a Project share the same infrastructure and the Scheduler.

- +belongs to a Project. All Namespaces of a Project share the same infrastructure and the Scheduler.

+ \ No newline at end of file diff --git a/docs/concepts/plugin/index.html b/docs/concepts/plugin/index.html index c279506b21..1e6a627375 100644 --- a/docs/concepts/plugin/index.html +++ b/docs/concepts/plugin/index.html @@ -10,7 +10,7 @@ - + @@ -18,8 +18,8 @@

Plugin

Optimus can provide support for various data warehouses & any third party system to handle all the data transformations or movement through plugins. You can bring your own plugin by encapsulating all the logic in a docker container.

Currently, plugins can be defined as YAML or binary executables. YAML plugin provides the questionnaire and default values for job task’s / hook’s creation, as well as defines the image to execute. While a binary plugin, it is -complementing the YAML plugin by providing support for automated dependency resolution.

- +complementing the YAML plugin by providing support for automated dependency resolution.

+ \ No newline at end of file diff --git a/docs/concepts/project/index.html b/docs/concepts/project/index.html index 4a8d1d83c5..045b4151a0 100644 --- a/docs/concepts/project/index.html +++ b/docs/concepts/project/index.html @@ -10,14 +10,14 @@ - +

Project

A project/tenant represents a group of jobs, resources, and a scheduler with the specified configurations and infrastructure. -A project contains multiple user-created namespaces, and each has various jobs and resources.

- +A project contains multiple user-created namespaces, and each has various jobs and resources.

+ \ No newline at end of file diff --git a/docs/concepts/replay-and-backup/index.html b/docs/concepts/replay-and-backup/index.html index b4e91ffcbe..6a5dbe297c 100644 --- a/docs/concepts/replay-and-backup/index.html +++ b/docs/concepts/replay-and-backup/index.html @@ -10,7 +10,7 @@ - + @@ -20,8 +20,8 @@ the job tasks.

When validating, Optimus checks if there is any Replay with the same job and date currently running and also checks if the task scheduler instances are still running to avoid any duplication and conflicts.

After passing the validation checks, a Replay request will be created and will be processed by the workers based on the mode chosen (sequential/parallel). To re-run the tasks, Optimus clears the existing runs from the scheduler.

Sequential (Default)

Sequential Mode Flow

Parallel

Parallel Mode Flow

Optimus also provides a Backup feature to duplicate a resource that can be perfectly used before running Replay. Where -the backup result will be located, and the expiry detail can be configured in the project configuration.

- +the backup result will be located, and the expiry detail can be configured in the project configuration.

+ \ No newline at end of file diff --git a/docs/concepts/resource/index.html b/docs/concepts/resource/index.html index abb5038851..19df269014 100644 --- a/docs/concepts/resource/index.html +++ b/docs/concepts/resource/index.html @@ -10,7 +10,7 @@ - + @@ -18,8 +18,8 @@

Resource

A resource is the representation of the warehouse unit that can be a source or a destination of a transformation job. The warehouse resources can be created, modified, and be read from Optimus, as well as can be backed up as requested. Each warehouse supports a fixed set of resource types and each type has its own specification schema. -Optimus’ managed warehouse is called Optimus datastore.

At the moment, Optimus supports BigQuery datastore for these type of resources:

  • Dataset
  • Table
  • Standard View
  • External Table

Note: BigQuery resource deletion is currently not supported.

- +Optimus’ managed warehouse is called Optimus datastore.

At the moment, Optimus supports BigQuery datastore for these type of resources:

  • Dataset
  • Table
  • Standard View
  • External Table

Note: BigQuery resource deletion is currently not supported.

+ \ No newline at end of file diff --git a/docs/concepts/secret/index.html b/docs/concepts/secret/index.html index 28bcc83dba..64b515790a 100644 --- a/docs/concepts/secret/index.html +++ b/docs/concepts/secret/index.html @@ -10,7 +10,7 @@ - + @@ -20,8 +20,8 @@ to store secrets and make them accessible in containers during the execution.

You can easily create, update, and delete your own secrets using CLI or REST API. Secrets can be created at a project level which is accessible from all the namespaces in the project, or can just be created at the namespace level. These secrets will then can be used as part of the job spec configuration using macros with their names. Only the secrets -created at the project & namespace the job belongs to can be referenced.

- +created at the project & namespace the job belongs to can be referenced.

+ \ No newline at end of file diff --git a/docs/contribute/contribution-process/index.html b/docs/contribute/contribution-process/index.html index d0099c4778..5c4ce14d57 100644 --- a/docs/contribute/contribution-process/index.html +++ b/docs/contribute/contribution-process/index.html @@ -10,7 +10,7 @@ - + @@ -19,8 +19,8 @@ You are also welcome to create issue if you encounter a bug or to suggest feature enhancements.

Please note we have a code of conduct, please follow it in all your interactions with the project.

Best practices​

  • Follow the conventional commit format for all commit messages.
  • Link the PR with the issue. This is mandatory to ensure there is sufficient information for the reviewer to understand your PR.
  • When you make a PR for small change (such as fixing a typo, style change, or grammar fix), please squash your commits so that we can maintain a cleaner git history.
  • Docs live in the code repo under docs. Please maintain the docs -and any docs changes can be done in the same PR.
  • Avoid force-pushing as it makes reviewing difficult.

Code of Conduct​

Examples of behavior that contributes to creating a positive environment include:

  • Using welcoming and inclusive language
  • Being respectful of differing viewpoints and experiences
  • Gracefully accepting constructive criticism
  • Focusing on what is best for the project

Things to keep in mind before creating a new commit:

- +and any docs changes can be done in the same PR.
  • Avoid force-pushing as it makes reviewing difficult.

    • Code of Conduct​

    Examples of behavior that contributes to creating a positive environment include:

    • Using welcoming and inclusive language
    • Being respectful of differing viewpoints and experiences
    • Gracefully accepting constructive criticism
    • Focusing on what is best for the project

    Things to keep in mind before creating a new commit:

    + \ No newline at end of file diff --git a/docs/contribute/developer-env-setup/index.html b/docs/contribute/developer-env-setup/index.html index a643021fb1..42658e8841 100644 --- a/docs/contribute/developer-env-setup/index.html +++ b/docs/contribute/developer-env-setup/index.html @@ -10,14 +10,14 @@ - +

    Developer Environment Setup

    To test Optimus including the integration with Airflow, as well as testing the runtime interactions, Optimus provides -a way to simplify setting up the environment. Take a look and follow the guide in dev setup section.

    - +a way to simplify setting up the environment. Take a look and follow the guide in dev setup section.

    + \ No newline at end of file diff --git a/docs/getting-started/installation/index.html b/docs/getting-started/installation/index.html index 7a8cc58267..0aba3efde7 100644 --- a/docs/getting-started/installation/index.html +++ b/docs/getting-started/installation/index.html @@ -10,13 +10,13 @@ - +
    -

    Installation

    Installing Optimus on any system is straight forward. There are several approaches to install Optimus:

    • Using a pre-built binary
    • Installing with package manager
    • Installing with Docker
    • Installing from source

    Using a Pre-built Binary​

    The client and server binaries are downloadable at the releases section.

    Once installed, you should be able to run:

    $ optimus version

    Installing with Package Manager​

    For macOS, you can install Optimus using homebrew:

    $ brew install goto/tap/optimus
    $ optimus version

    Installing using Docker​

    To pull latest image:

    $ docker pull goto/optimus:latest

    To pull specific image:

    $ docker pull goto/optimus:0.6.0

    Installing from Source​

    Prerequisites​

    Optimus requires the following dependencies:

    • Golang (version 1.18 or above)
    • Git

    Build​

    Run the following commands to compile optimus from source

    $ git clone git@github.com:goto/optimus.git
    $ cd optimus
    $ make build

    Use the following command to test

    $ optimus version
    - +

    Installation

    Installing Optimus on any system is straight forward. There are several approaches to install Optimus:

    • Using a pre-built binary
    • Installing with package manager
    • Installing with Docker
    • Installing from source

    Using a Pre-built Binary​

    The client and server binaries are downloadable at the releases section.

    Once installed, you should be able to run:

    $ optimus version

    Installing with Package Manager​

    For macOS, you can install Optimus using homebrew:

    $ brew install goto/tap/optimus
    $ optimus version

    Installing using Docker​

    To pull latest image:

    $ docker pull goto/optimus:latest

    To pull specific image:

    $ docker pull goto/optimus:0.6.0

    Installing from Source​

    Prerequisites​

    Optimus requires the following dependencies:

    • Golang (version 1.18 or above)
    • Git

    Build​

    Run the following commands to compile optimus from source

    $ git clone git@github.com:goto/optimus.git
    $ cd optimus
    $ make build

    Use the following command to test

    $ optimus version
    + \ No newline at end of file diff --git a/docs/getting-started/quick-start/index.html b/docs/getting-started/quick-start/index.html index 70d70fdc48..95cd9ae660 100644 --- a/docs/getting-started/quick-start/index.html +++ b/docs/getting-started/quick-start/index.html @@ -10,7 +10,7 @@ - + @@ -29,8 +29,8 @@ of job specification and the possible options here.

    Before proceeding, let’s add the BQ_SERVICE_ACCOUNT secret in the task configuration.

    task:
      name: bq2bq
      config:
        BQ_SERVICE_ACCOUNT: "{{.secret.BQ_SERVICE_ACCOUNT}}"
        DATASET: sample_namespace
    ...

    Later, you can avoid having the secret specified in every single job specification by adding it in the parent yaml specification instead. For more details, you can take a look here.

    Now the job specification has been prepared, lets try to add it to the server by running this command:

    $ optimus job replace-all --verbose

    > Validating namespaces
    validation finished!

    > Replacing all jobs for namespaces [sample_namespace]
    > Receiving responses:
    [sample_namespace] received 1 job specs
    [sample_namespace] found 1 new, 0 modified, and 0 deleted job specs
    [sample_namespace] processing job job1
    [sample_namespace] successfully added 1 jobs
    replace all job specifications finished!

    Above command will try to add/modify all job specifications found in your project. We are not providing registering a single job through Optimus CLI, but it is possible to do so using API.

    Now that the jobs has been registered to Optimus, let’s compile and upload it to the scheduler by using the following command.

    $ optimus scheduler upload-all

    The command will try to compile your job specification to the DAG file. The result will be stored in the storage_path -location as you have specified when configuring the optimus.yaml file.

    Later, once you have Airflow ready and want to try out, this directory can be used as a source to be scheduled by Airflow.

    - +location as you have specified when configuring the optimus.yaml file.

    Later, once you have Airflow ready and want to try out, this directory can be used as a source to be scheduled by Airflow.

    + \ No newline at end of file diff --git a/docs/introduction/index.html b/docs/introduction/index.html index bf3cc2dfc1..9e1de7d912 100644 --- a/docs/introduction/index.html +++ b/docs/introduction/index.html @@ -10,7 +10,7 @@ - + @@ -29,8 +29,8 @@ defining the same. The dependencies are managed across tenants, so teams doesn’t need to coordinate among themselves.

    In-Built Alerting​

    Always get notified when your job is not behaving as expected, on failures or on SLA misses. Optimus supports integrations with slack & pagerduty.

    Verification in Advance​

    Minimize job runtime issues by validating and inspecting jobs before submitting them which enables them for faster turnaround time when submitting their jobs. Users can get to know about job dependencies, validation failures & some -warnings before submitting the jobs.

    - +warnings before submitting the jobs.

    + \ No newline at end of file diff --git a/docs/reference/api/index.html b/docs/reference/api/index.html index e00861381e..d24e3b3d51 100644 --- a/docs/reference/api/index.html +++ b/docs/reference/api/index.html @@ -10,13 +10,13 @@ - +
    -

    API

    Optimus service supports REST and GRPC for interaction, currently on the same port.

    - +

    API

    Optimus service supports REST and GRPC for interaction, currently on the same port.

    + \ No newline at end of file diff --git a/docs/reference/faq/index.html b/docs/reference/faq/index.html index 86dc5fb9af..eacd67ebcc 100644 --- a/docs/reference/faq/index.html +++ b/docs/reference/faq/index.html @@ -10,7 +10,7 @@ - + @@ -22,8 +22,8 @@ have a hook without a transformation task.

  • I have a job with a modified view source, however, it does not detect as modified when running the job replace-all command. How should I apply the change?

    It does not detect as modified as the specification and the assets of the job itself is not changed. Do run the job refresh command.

  • My job is failing due to the resource is not sufficient. Can I scale a specific job to have a bigger CPU / memory limit?

    Yes, resource requests and limits are configurable in the job specification’s metadata field.

  • I removed a resource specification, but why does the resource is not deleted in the BigQuery project?

    Optimus currently does not support resource deletion to avoid any accidental deletion.

  • I removed a job specification, but why does it still appear in Airflow UI?

    You might want to check the log of the replace-all command you are using. It will show whether your job has been successfully deleted or not. If not, the possible causes are the job is being used by another job as a dependency. -You can force delete it through API if needed.

    If the job has been successfully deleted, it might take time for the deletion to reflect which depends on your Airflow sync or DAG load configuration.

    • - +You can force delete it through API if needed.

    If the job has been successfully deleted, it might take time for the deletion to reflect which depends on your Airflow sync or DAG load configuration.

    + \ No newline at end of file diff --git a/docs/reference/metrics/index.html b/docs/reference/metrics/index.html index 13318b4c8c..f93d12570d 100644 --- a/docs/reference/metrics/index.html +++ b/docs/reference/metrics/index.html @@ -10,13 +10,13 @@ - +
    -

    Metrics

    Job Change Metrics​

    NameTypeDescriptionLabels
    job_events_totalcounterNumber of job changes attempt.project, namespace, status
    job_upload_totalcounterNumber of jobs uploaded to scheduler.project, namespace, status
    job_removal_totalcounterNumber of jobs removed from scheduler.project, namespace, status
    job_namespace_migrations_totalcounterNumber of jobs migrated from a namespace to another.project, namespace_source, namespace_destination
    job_replace_all_duration_secondscounterDuration of job 'replace-all' process in seconds.project, namespace
    job_refresh_duration_secondscounterDuration of job 'refresh' process in seconds.project
    job_validation_duration_secondscounterDuration of job 'validation' process in seconds.project, namespace

    JobRun Metrics​

    NameTypeDescriptionLabels
    jobrun_events_totalcounterNumber of jobrun events in a job broken by the status, e.g sla_miss, wait_upstream, in_progress, success, failed.project, namespace, job, status
    jobrun_sensor_events_totalcounterNumber of sensor run events broken by the event_type, e.g start, retry, success, fail.project, namespace, event_type
    jobrun_task_events_totalcounterNumber of task run events for a given operator (task name) broken by the event_type, e.g start, retry, success, fail.project, namespace, event_type, operator
    jobrun_hook_events_totalcounterNumber of hook run events for a given operator (task name) broken by the event_type, e.g start, retry, success, fail.project, namespace, event_type, operator
    jobrun_replay_requests_totalcounterNumber of replay requests for a single job.project, namespace, job, status
    jobrun_alerts_totalcounterNumber of the alerts triggered broken by the alert type.project, namespace, type

    Resource Metrics​

    NameTypeDescriptionLabels
    resource_events_totalcounterNumber of resource change attempts broken down by the resource type and status.project, namespace, datastore, type, status
    resource_namespace_migrations_totalcounterNumber of resources migrated from a namespace to another namespace.project, namespace_source, namespace_destination
    resource_upload_all_duration_secondsgaugeDuration of uploading all resource specification in seconds.project, namespace
    resource_backup_requests_totalcounterNumber of backup requests for a single resource.project, namespace, resource, status

    Tenant Metrics​

    NameTypeDescriptionLabels
    secret_events_totalcounterNumber of secret change attempts.project, namespace, status

    System Metrics​

    NameTypeDescriptionLabels
    application_heartbeatcounterOptimus server heartbeat pings.-
    application_uptime_secondsgaugeSeconds since the application started.-
    notification_queue_totalcounterNumber of items queued in the notification channel.type
    notification_worker_batch_totalcounterNumber of worker executions in the notification channel.type
    notification_worker_send_err_totalcounterNumber of events created and to be sent to writer.type
    publisher_kafka_events_queued_totalcounterNumber of events queued to be published to kafka topic.-
    - +

    Metrics

    Job Change Metrics​

    NameTypeDescriptionLabels
    job_events_totalcounterNumber of job changes attempt.project, namespace, status
    job_upload_totalcounterNumber of jobs uploaded to scheduler.project, namespace, status
    job_removal_totalcounterNumber of jobs removed from scheduler.project, namespace, status
    job_namespace_migrations_totalcounterNumber of jobs migrated from a namespace to another.project, namespace_source, namespace_destination
    job_replace_all_duration_secondscounterDuration of job 'replace-all' process in seconds.project, namespace
    job_refresh_duration_secondscounterDuration of job 'refresh' process in seconds.project
    job_validation_duration_secondscounterDuration of job 'validation' process in seconds.project, namespace

    JobRun Metrics​

    NameTypeDescriptionLabels
    jobrun_events_totalcounterNumber of jobrun events in a job broken by the status, e.g sla_miss, wait_upstream, in_progress, success, failed.project, namespace, job, status
    jobrun_sensor_events_totalcounterNumber of sensor run events broken by the event_type, e.g start, retry, success, fail.project, namespace, event_type
    jobrun_task_events_totalcounterNumber of task run events for a given operator (task name) broken by the event_type, e.g start, retry, success, fail.project, namespace, event_type, operator
    jobrun_hook_events_totalcounterNumber of hook run events for a given operator (task name) broken by the event_type, e.g start, retry, success, fail.project, namespace, event_type, operator
    jobrun_replay_requests_totalcounterNumber of replay requests for a single job.project, namespace, job, status
    jobrun_alerts_totalcounterNumber of the alerts triggered broken by the alert type.project, namespace, type

    Resource Metrics​

    NameTypeDescriptionLabels
    resource_events_totalcounterNumber of resource change attempts broken down by the resource type and status.project, namespace, datastore, type, status
    resource_namespace_migrations_totalcounterNumber of resources migrated from a namespace to another namespace.project, namespace_source, namespace_destination
    resource_upload_all_duration_secondsgaugeDuration of uploading all resource specification in seconds.project, namespace
    resource_backup_requests_totalcounterNumber of backup requests for a single resource.project, namespace, resource, status

    Tenant Metrics​

    NameTypeDescriptionLabels
    secret_events_totalcounterNumber of secret change attempts.project, namespace, status

    System Metrics​

    NameTypeDescriptionLabels
    application_heartbeatcounterOptimus server heartbeat pings.-
    application_uptime_secondsgaugeSeconds since the application started.-
    notification_queue_totalcounterNumber of items queued in the notification channel.type
    notification_worker_batch_totalcounterNumber of worker executions in the notification channel.type
    notification_worker_send_err_totalcounterNumber of events created and to be sent to writer.type
    publisher_kafka_events_queued_totalcounterNumber of events queued to be published to kafka topic.-
    + \ No newline at end of file diff --git a/docs/rfcs/improving_time_and_flow_of_deployment/index.html b/docs/rfcs/improving_time_and_flow_of_deployment/index.html index f7ff1b8a00..3705bdb8c7 100644 --- a/docs/rfcs/improving_time_and_flow_of_deployment/index.html +++ b/docs/rfcs/improving_time_and_flow_of_deployment/index.html @@ -10,7 +10,7 @@ - + @@ -52,8 +52,8 @@ what is the transformation destination of the job. However, we are not storing what are the sources of the transformation. The only thing we have is job dependency, not resource.

    We can add a Source URNs field to the jobs specs, or create a Job Source table. Whenever there is a change in a view through Optimus, datastore should be able to request the dependency resolution for the view's dependent and having the -dependencies updated. We will also provide the mechanism to refresh jobs.

    CLI Perspective​

    Deploy job per namespace (using DeployJobSpecification rpc)

    optimus job deploy --namespace --project

    Deploy job for selected jobs (using CreateJobSpecification rpc)

    optimus job deploy --namespace --project --jobs=(job1,job2)

    Refresh the entire namespace/project

    optimus job refresh --namespace --project

    Refresh the selected jobs

    optimus job refresh --namespace --project --jobs=(job1,job2)

    Other Thoughts:​

    Cache Considerations instead of persisting to PG

    • Might be faster as there is no additional time for write/read from DB
    • Data will be unavailable post pod restarts. Need to redo the dependency resolution overall
    • Poor visibility
    - +dependencies updated. We will also provide the mechanism to refresh jobs.

    CLI Perspective​

    Deploy job per namespace (using DeployJobSpecification rpc)

    optimus job deploy --namespace --project

    Deploy job for selected jobs (using CreateJobSpecification rpc)

    optimus job deploy --namespace --project --jobs=(job1,job2)

    Refresh the entire namespace/project

    optimus job refresh --namespace --project

    Refresh the selected jobs

    optimus job refresh --namespace --project --jobs=(job1,job2)

    Other Thoughts:​

    Cache Considerations instead of persisting to PG

    • Might be faster as there is no additional time for write/read from DB
    • Data will be unavailable post pod restarts. Need to redo the dependency resolution overall
    • Poor visibility
    + \ No newline at end of file diff --git a/docs/rfcs/index.html b/docs/rfcs/index.html index 3df851fb30..352934c54a 100644 --- a/docs/rfcs/index.html +++ b/docs/rfcs/index.html @@ -10,13 +10,13 @@ - +
    -

    About This Directory

    This directory contains RFCs (design documents) that describe proposed major changes to Optimus.

    The Why of RFCs

    An RFC provides a high-level description of a major change or enhancement to Optimus. The high-level description allows a reviewer to critique and poke holes in a design without getting lost in the particulars of code.

    An RFC is a form of communication aimed at both spreading and gathering knowledge, though it is not the sole means of accomplishing either task. Prototypes, tech notes, github issues, comments in code, commit messages and in-person discussions are valid alternatives depending on the situation.

    At its best, an RFC clearly and concisely describes the high-level architecture of a project giving confidence to all involved. At its worst, an RFC focuses on unimportant details, fosters discussion that stymies progress, or demoralizes the author with the complexity of their undertaking.

    The When and How of RFCs

    When to write an RFC is nuanced and there are no firm rules. General guidance is to write an RFC before embarking on a significant or complex project that will be spread over multiple pull requests (PRs), and when multiple alternatives need to be considered and there is no obvious best approach. A project involving multiple people is a good signal an RFC is warranted. (Similarly, a project worthy of an RFC often requires multiple engineers worth of effort). Note that this guidance is intentionally vague. Many complex projects spread over multiple PRs do not require an RFC, though they do require adequate communication and discussion.

    It is encouraged to develop a prototype concurrently with writing the RFC. One of the significant benefits of an RFC is that it forces bigger picture thinking which reviewers can then disect. In contrast, a prototype forces the details to be considered, shedding light on the unknown and helping to ensure that the RFC focuses on the important design considerations.

    An RFC should be a high-level description which does not require formal correctness. There is utility in conciseness. Do not overspecify the details in the RFC as doing so can bury the reviewer in minutiae. If you've never written an RFC before consider partnering with a more experienced engineer for guidance and to help shepherd your RFC through the process.

    RFC Process

    1. Every RFC should have a dedicated reviewer familiar with the RFC's subject area.

    2. Copy 00000000_template.md to a new file and fill in the details. Commit this version in your own fork of the repository or a branch. Your commit message (and corresponding pull request) should include the prefix rfc. Eg: rfc: edit RFC template

      If you are a creative person, you may prefer to start with this blank slate, write your prose and then later check that all necessary topics have been covered.

      If you feel intimidated by a blank template, you can instead peruse the list of requested topics and use the questions in there as writing prompt.

    3. Submit a pull request (PR) to add your new file to the main repository. Each RFC should get its own pull request; do not combine RFCs with other files.

      Note: you can send a PR before the RFC is complete in order to solicit input about what to write in the RFC. In this case, include the term "[WIP]" (work in progress) in the PR title & mark the PR as draft until you are confident the RFC is complete and can be reviewed.

    4. Go through the PR review, iterating on the RFC to answer questions and concerns from the reviewer(s). The duration of this process should be related to the complexity of the project. If you or the reviewers seem to be at an impasse, consider in-person discussions or a prototype. There is no minimum time required to leave an RFC open for review. There is also no prohibition about halting or reversing work on an accepted RFC if a problem is discovered during implementation.

      Reviewers should be conscious of their own limitations and ask for other engineers to look at specific areas if necessary.

    5. Once discussion has settled and the RFC has received an LGTM from the reviewer(s):

      • change the Status field of the document to in-progress;
      • rename the RFC document to prefix it with the current date (YYYYMMDD_);
      • update the RFC PR field;
      • and merge the PR.

    6. Once the changes in the RFC have been implemented and merged, change the Status field of the document from in-progress to completed. If subsequent developments render an RFC obsolete, change its status to obsolete. When you mark a RFC as obsolete, ensure that its text references the other RFCs or PRs that make it obsolete.

    - +

    About This Directory

    This directory contains RFCs (design documents) that describe proposed major changes to Optimus.

    The Why of RFCs

    An RFC provides a high-level description of a major change or enhancement to Optimus. The high-level description allows a reviewer to critique and poke holes in a design without getting lost in the particulars of code.

    An RFC is a form of communication aimed at both spreading and gathering knowledge, though it is not the sole means of accomplishing either task. Prototypes, tech notes, github issues, comments in code, commit messages and in-person discussions are valid alternatives depending on the situation.

    At its best, an RFC clearly and concisely describes the high-level architecture of a project giving confidence to all involved. At its worst, an RFC focuses on unimportant details, fosters discussion that stymies progress, or demoralizes the author with the complexity of their undertaking.

    The When and How of RFCs

    When to write an RFC is nuanced and there are no firm rules. General guidance is to write an RFC before embarking on a significant or complex project that will be spread over multiple pull requests (PRs), and when multiple alternatives need to be considered and there is no obvious best approach. A project involving multiple people is a good signal an RFC is warranted. (Similarly, a project worthy of an RFC often requires multiple engineers worth of effort). Note that this guidance is intentionally vague. Many complex projects spread over multiple PRs do not require an RFC, though they do require adequate communication and discussion.

    It is encouraged to develop a prototype concurrently with writing the RFC. One of the significant benefits of an RFC is that it forces bigger picture thinking which reviewers can then disect. In contrast, a prototype forces the details to be considered, shedding light on the unknown and helping to ensure that the RFC focuses on the important design considerations.

    An RFC should be a high-level description which does not require formal correctness. There is utility in conciseness. Do not overspecify the details in the RFC as doing so can bury the reviewer in minutiae. If you've never written an RFC before consider partnering with a more experienced engineer for guidance and to help shepherd your RFC through the process.

    RFC Process

    1. Every RFC should have a dedicated reviewer familiar with the RFC's subject area.

    2. Copy 00000000_template.md to a new file and fill in the details. Commit this version in your own fork of the repository or a branch. Your commit message (and corresponding pull request) should include the prefix rfc. Eg: rfc: edit RFC template

      If you are a creative person, you may prefer to start with this blank slate, write your prose and then later check that all necessary topics have been covered.

      If you feel intimidated by a blank template, you can instead peruse the list of requested topics and use the questions in there as writing prompt.

    3. Submit a pull request (PR) to add your new file to the main repository. Each RFC should get its own pull request; do not combine RFCs with other files.

      Note: you can send a PR before the RFC is complete in order to solicit input about what to write in the RFC. In this case, include the term "[WIP]" (work in progress) in the PR title & mark the PR as draft until you are confident the RFC is complete and can be reviewed.

    4. Go through the PR review, iterating on the RFC to answer questions and concerns from the reviewer(s). The duration of this process should be related to the complexity of the project. If you or the reviewers seem to be at an impasse, consider in-person discussions or a prototype. There is no minimum time required to leave an RFC open for review. There is also no prohibition about halting or reversing work on an accepted RFC if a problem is discovered during implementation.

      Reviewers should be conscious of their own limitations and ask for other engineers to look at specific areas if necessary.

    5. Once discussion has settled and the RFC has received an LGTM from the reviewer(s):

      • change the Status field of the document to in-progress;
      • rename the RFC document to prefix it with the current date (YYYYMMDD_);
      • update the RFC PR field;
      • and merge the PR.

    6. Once the changes in the RFC have been implemented and merged, change the Status field of the document from in-progress to completed. If subsequent developments render an RFC obsolete, change its status to obsolete. When you mark a RFC as obsolete, ensure that its text references the other RFCs or PRs that make it obsolete.

    + \ No newline at end of file diff --git a/docs/rfcs/optimus_dashboarding/index.html b/docs/rfcs/optimus_dashboarding/index.html index 58e2e1d5f2..5e91c827af 100644 --- a/docs/rfcs/optimus_dashboarding/index.html +++ b/docs/rfcs/optimus_dashboarding/index.html @@ -10,13 +10,13 @@ - +
    -

    optimus_dashboarding

    • Feature Name: Optimus Dashboarding
    • Status: Draft
    • Start Date: 2022-05-17
    • Authors: Yash Bhardwaj

    Summary

    Optimus users need to frequestly visit airflow webserver which does not provide a optimus perspective of the jobs i.e. is lacks the leniage view and project / namespace level clasification.

    The proposal here is to capture job related events and state information via airflow event callbacks and job lifecycle events .

    Design

    Job Metrics dashboard (aka. Job sumaries page):​

    • Monitor jobs performance stats :
      • Duaration per optimus job task/sensor/hook.
      • Job completion Sla misses trend.
      • Job summaries.
    • Jobs groupings into namespaces and projects.

    Job lineage view :​

    • Currently only columbus provides lineage view.
    • That does not reflect the job status and run level dependencies/relationships.

    Approach : Collect info from Airflow events / callbacks :​

    1. Get DAG lifecycle events from scheduler through:

      • Airflow triggered Callbacks
        • on_success_callback Invoked when the task succeeds
        • on_failure_callback Invoked when the task fails
        • sla_miss_callback Invoked when a task misses its defined SLA ( SLA here is scheduling delay not to be concused with job completion delay )
        • on_retry_callback Invoked when the task is up for retry
      • Events fired by our Custom logic added into
        • Custom operator implimentation
        • airflow job_start_event and job_end_event task

    2. Information from these events is then relayed to the optimus server. Optimus then writes this into

    • OptimusDB : for summaries dashboarding and for powering lineage views (in future)
    1. Reasons for choosing this approach
    • This is less tightly coupled with the Current chosen Scheduler.
      • If later support is provided for more schedulers, exixting optimus data collection APIs and Optimus Data model can be reused to power our Frontend system.
    1. Known limitations:
    • Since this is an event based architecture, collection of Job/DAG state info will be hampered in cases of panic failures for instance DAG python code throwing uncaught exception.
      • even in such cases we will be able to determine the SLA missing jobs

    Optimus Perspective DB model:​

    • Optimus shall consider each scheduled run and its subsiquent retry as a new job_run
    • sensor/task/hooks run information shall be grouped per job_run
      • To achive this, each sensor/task/hook Run is linked with the job_run.id and job_run.attempt_number
      • while registering a sensor run the latest job_run.attempt for that given schedule time is used to link it.
    • For each job_run(scheduled/re-run) there will only be one row per each sensor/task/hook registered in the Optimus DB.

    Compute SLA miss:​

    SLA Definition :​

    • if Time duration between the job execution start time and job completion time excedes the Defined SLA Limit, then that is termed as an SLA breach.

    Approach For SLA dashboarding:​

    • Note Job Start Time
    • With Each Job Run, associate the then SLA definition(user defined number in the job.yaml) of the job.
    • SLA breach are determined with the read QUERY of Grafana, which works as following
      • SLA breach duration = (min(job_end_time , time.now()) - job_start_time) - SLA_definition
    • Limitations
      • Since this is an event based data collection setup is it possible that a job may have failed/crashed/hangged in such a situation optimus wont get the job finish callback, and hence cant determine the SLA breach
        • To work arround with that, at the time of job run registeration the end time is assumed to be a far future date. In case the job terminates properly we shall be able to determine the correct end_time , otherwise optimus is safe to assume that the job has not finised yet. The job Duration in such case will be the the time since the job has started running.

    Approach For SLA alerting:​

    • Ariflow DAG level SLA definiation can be used to get DAG SLA breach event.
      • At the end of each task, a check is conducted to test whether the completed task’s end-time exceeded the SLA OR the start time of the next task exceeded the SLA.

    Other Considerations:​

    • contrary approached discussed
      • Kubernetes Sidecar
        • SideCar lifecycle hooks start/end
        • sideCar to pull details from scheduler/plugin containers and push same to optimus server
      • Pull Approach
        • Callbacks -> statsD to power job sumaries page
        • access airflow API directly from Optimus to power job details view
    • Future considerations
      • Support for Events fired by Executor :
        • it is expected that even optimus-task and hooks shall independently be able to emit events to optimus notify api. this should help with improved executor observability.

    Terminology:​

    • Task Airflow task operator
    • Job Optimus job
    • DAG Airflow DAG
    - +

    optimus_dashboarding

    • Feature Name: Optimus Dashboarding
    • Status: Draft
    • Start Date: 2022-05-17
    • Authors: Yash Bhardwaj

    Summary

    Optimus users need to frequestly visit airflow webserver which does not provide a optimus perspective of the jobs i.e. is lacks the leniage view and project / namespace level clasification.

    The proposal here is to capture job related events and state information via airflow event callbacks and job lifecycle events .

    Design

    Job Metrics dashboard (aka. Job sumaries page):​

    • Monitor jobs performance stats :
      • Duaration per optimus job task/sensor/hook.
      • Job completion Sla misses trend.
      • Job summaries.
    • Jobs groupings into namespaces and projects.

    Job lineage view :​

    • Currently only columbus provides lineage view.
    • That does not reflect the job status and run level dependencies/relationships.

    Approach : Collect info from Airflow events / callbacks :​

    1. Get DAG lifecycle events from scheduler through:

      • Airflow triggered Callbacks
        • on_success_callback Invoked when the task succeeds
        • on_failure_callback Invoked when the task fails
        • sla_miss_callback Invoked when a task misses its defined SLA ( SLA here is scheduling delay not to be concused with job completion delay )
        • on_retry_callback Invoked when the task is up for retry
      • Events fired by our Custom logic added into
        • Custom operator implimentation
        • airflow job_start_event and job_end_event task

    2. Information from these events is then relayed to the optimus server. Optimus then writes this into

    • OptimusDB : for summaries dashboarding and for powering lineage views (in future)
    1. Reasons for choosing this approach
    • This is less tightly coupled with the Current chosen Scheduler.
      • If later support is provided for more schedulers, exixting optimus data collection APIs and Optimus Data model can be reused to power our Frontend system.
    1. Known limitations:
    • Since this is an event based architecture, collection of Job/DAG state info will be hampered in cases of panic failures for instance DAG python code throwing uncaught exception.
      • even in such cases we will be able to determine the SLA missing jobs

    Optimus Perspective DB model:​

    • Optimus shall consider each scheduled run and its subsiquent retry as a new job_run
    • sensor/task/hooks run information shall be grouped per job_run
      • To achive this, each sensor/task/hook Run is linked with the job_run.id and job_run.attempt_number
      • while registering a sensor run the latest job_run.attempt for that given schedule time is used to link it.
    • For each job_run(scheduled/re-run) there will only be one row per each sensor/task/hook registered in the Optimus DB.

    Compute SLA miss:​

    SLA Definition :​

    • if Time duration between the job execution start time and job completion time excedes the Defined SLA Limit, then that is termed as an SLA breach.

    Approach For SLA dashboarding:​

    • Note Job Start Time
    • With Each Job Run, associate the then SLA definition(user defined number in the job.yaml) of the job.
    • SLA breach are determined with the read QUERY of Grafana, which works as following
      • SLA breach duration = (min(job_end_time , time.now()) - job_start_time) - SLA_definition
    • Limitations
      • Since this is an event based data collection setup is it possible that a job may have failed/crashed/hangged in such a situation optimus wont get the job finish callback, and hence cant determine the SLA breach
        • To work arround with that, at the time of job run registeration the end time is assumed to be a far future date. In case the job terminates properly we shall be able to determine the correct end_time , otherwise optimus is safe to assume that the job has not finised yet. The job Duration in such case will be the the time since the job has started running.

    Approach For SLA alerting:​

    • Ariflow DAG level SLA definiation can be used to get DAG SLA breach event.
      • At the end of each task, a check is conducted to test whether the completed task’s end-time exceeded the SLA OR the start time of the next task exceeded the SLA.

    Other Considerations:​

    • contrary approached discussed
      • Kubernetes Sidecar
        • SideCar lifecycle hooks start/end
        • sideCar to pull details from scheduler/plugin containers and push same to optimus server
      • Pull Approach
        • Callbacks -> statsD to power job sumaries page
        • access airflow API directly from Optimus to power job details view
    • Future considerations
      • Support for Events fired by Executor :
        • it is expected that even optimus-task and hooks shall independently be able to emit events to optimus notify api. this should help with improved executor observability.

    Terminology:​

    • Task Airflow task operator
    • Job Optimus job
    • DAG Airflow DAG
    + \ No newline at end of file diff --git a/docs/rfcs/replay_rate_limiting/index.html b/docs/rfcs/replay_rate_limiting/index.html index b7306cd80f..9007c67512 100644 --- a/docs/rfcs/replay_rate_limiting/index.html +++ b/docs/rfcs/replay_rate_limiting/index.html @@ -10,7 +10,7 @@ - + @@ -32,8 +32,8 @@ can take another runs.

    Picking which runs to be processed:

    • Based on Replay request time (FIFO)
    • Based on the slot per dag. If the overall is set to 15 but the slot per dag is only 5, then a request to replay a job that has 10 runs cannot be done in one go.

    There should be a default value for each of the configuration.

    • Max_replay_runs_per_project: default 10
    • Max_replay_runs_per_dag: default 5

    Optimus will check on how many Replay job is currently running, and how many task run is running for each job. Additional changes might be needed on Replay side for optimization, to avoid calling Airflow API in every interval (Replay syncer -responsibility) and fetch data from Replay table instead.

    - +responsibility) and fetch data from Replay table instead.

    + \ No newline at end of file diff --git a/docs/rfcs/revisit_automated_dependency_resolution/index.html b/docs/rfcs/revisit_automated_dependency_resolution/index.html index 376af693a5..cddc16afdd 100644 --- a/docs/rfcs/revisit_automated_dependency_resolution/index.html +++ b/docs/rfcs/revisit_automated_dependency_resolution/index.html @@ -10,13 +10,13 @@ - +
    -

    revisit_automated_dependency_resolution

    • Feature Name: Revisit Automated Dependency Resolution Logic
    • Status: Draft
    • Start Date: 2022-01-24
    • Authors:

    Summary

    Optimus is a data warehouse management system, data is at the core of Optimus. Automated dependency resolution is the core problem which Optimus wants to address. The current windowing is confusing, so lets take a revisit to understand how we can solve this problem if we are tackling it fresh.

    Technical Design

    Background :​

    Input Data Flows through the system & it is expected to arrive at some delay or user gives enough buffer for all late data to arrive. Post that, the user expects to schedule the job to process the data after the max delay.

    Keeping this basic idea in mind, what logic can be used to enable automated dependency resolution is the key question for us? And what all questions need to be answered for the same?

    Question 1 : What is the time range of data a job consumes from the primary sources?

    Question 2 : What is the time range of data a job writes?

    If these two questions be answered for every scheduled job then dependent jobs be computed accordingly.

    Approach :​

    Let's answer the Question 1, this is clearly a user input, there is no computation here. How intuitively a user input can be taken is the key here.

    data_window : 
     max_delay : 1d/2h/1d2h
     amount : 1d/1w/2d/1m

    max_delay is used to identify the end time of the window.

    amount is the amount of data the user is consuming from the primary sources.

    Below is the current windowing configuration.

      window:
        size: 24h
        offset: 24h
        truncate_to: d

    Let's answer Question 2, I believe this is mainly linked with the schedule of the job, if the job is scheduled daily then the expectation is the job makes data available for a whole day, if hourly then for a whole hour, irrespective of the input_window. What exactly is the time range can be computed by max_delay and schedule_frequency .

    If the job has a max_delay of 2h & the job is scheduled at 2 AM UTC then the job is making the data available for the entire previous day, irrespective of the window(1d,1m,1w).

    WIth this core idea there are few scenarios which should not be allowed or which cannot be used for automated depedency resolution to work in those cases the jobs just depend on the previous jobs for eg., dynamic schedules. If a job is scheduled only 1st,2nd & 3rd hour of the day.

    The next part of the solution is how to do the actual dependency resolution.

    1. Compute the data_window based on user input & the schedule time.
    2. Identify all the upstreams.
    3. Starting from the current schedule_time get each upstream's schedule in the past & future and compute the output data range till it finds runs which falls outside the window in each direction.

    Other Thoughts:​

    Inorder to keep things simple for most of the users, if a user doesn't define any window then the jobs depend on the immediate upstream by schedule_time for that job & as well as the jobs that are depending the above job.

    dstart & dend macros will still be offered to users which they can use for data filtering in their queries.

    How do we transition to this new approach?​

    1. Support Old & New approaches both, migrate all jobs to the new strategy & later cleanup the code.
    - +

    revisit_automated_dependency_resolution

    • Feature Name: Revisit Automated Dependency Resolution Logic
    • Status: Draft
    • Start Date: 2022-01-24
    • Authors:

    Summary

    Optimus is a data warehouse management system, data is at the core of Optimus. Automated dependency resolution is the core problem which Optimus wants to address. The current windowing is confusing, so lets take a revisit to understand how we can solve this problem if we are tackling it fresh.

    Technical Design

    Background :​

    Input Data Flows through the system & it is expected to arrive at some delay or user gives enough buffer for all late data to arrive. Post that, the user expects to schedule the job to process the data after the max delay.

    Keeping this basic idea in mind, what logic can be used to enable automated dependency resolution is the key question for us? And what all questions need to be answered for the same?

    Question 1 : What is the time range of data a job consumes from the primary sources?

    Question 2 : What is the time range of data a job writes?

    If these two questions be answered for every scheduled job then dependent jobs be computed accordingly.

    Approach :​

    Let's answer the Question 1, this is clearly a user input, there is no computation here. How intuitively a user input can be taken is the key here.

    data_window : 
     max_delay : 1d/2h/1d2h
     amount : 1d/1w/2d/1m

    max_delay is used to identify the end time of the window.

    amount is the amount of data the user is consuming from the primary sources.

    Below is the current windowing configuration.

      window:
        size: 24h
        offset: 24h
        truncate_to: d

    Let's answer Question 2, I believe this is mainly linked with the schedule of the job, if the job is scheduled daily then the expectation is the job makes data available for a whole day, if hourly then for a whole hour, irrespective of the input_window. What exactly is the time range can be computed by max_delay and schedule_frequency .

    If the job has a max_delay of 2h & the job is scheduled at 2 AM UTC then the job is making the data available for the entire previous day, irrespective of the window(1d,1m,1w).

    WIth this core idea there are few scenarios which should not be allowed or which cannot be used for automated depedency resolution to work in those cases the jobs just depend on the previous jobs for eg., dynamic schedules. If a job is scheduled only 1st,2nd & 3rd hour of the day.

    The next part of the solution is how to do the actual dependency resolution.

    1. Compute the data_window based on user input & the schedule time.
    2. Identify all the upstreams.
    3. Starting from the current schedule_time get each upstream's schedule in the past & future and compute the output data range till it finds runs which falls outside the window in each direction.

    Other Thoughts:​

    Inorder to keep things simple for most of the users, if a user doesn't define any window then the jobs depend on the immediate upstream by schedule_time for that job & as well as the jobs that are depending the above job.

    dstart & dend macros will still be offered to users which they can use for data filtering in their queries.

    How do we transition to this new approach?​

    1. Support Old & New approaches both, migrate all jobs to the new strategy & later cleanup the code.
    + \ No newline at end of file diff --git a/docs/rfcs/secret_management/index.html b/docs/rfcs/secret_management/index.html index c56fd6c5fb..dcc502b34c 100644 --- a/docs/rfcs/secret_management/index.html +++ b/docs/rfcs/secret_management/index.html @@ -10,15 +10,15 @@ - +

    secret_management

    • Feature Name: Secret Management
    • Status: Approved
    • Start Date: 2021-10-02
    • Authors: Kush Sharma & Sravan

    Summary

    A lot of transformation operations require credentials to execute, there is a need to have a convenient way to save secrets and then access them in containers during the execution. This secret may also be needed in plugin adapters to compute dependencies/compile assets/etc before the actual transformation even begin. This is currently done using registering a secret to optimus so that it can be accessed by plugins and Kubernetes opaque secret, a single secret per plugin, getting mounted in the container(i.e. not at individual job level).

    This can be solved by allowing users to register secret from Optimus CLI as a key value pair, storing them encrypted using a single key across all tenants.

    Technical Design

    To keep string literals as secret, it is a requirement Optimus keep them encrypted in database. Optimus Server Key is used to encrypt & decrypt the secret & will ensure the secret is encrypted at rest. Each secret is a key value pair where key is an alpha numeric literal and value is base64 encoded string.

    Optimus has two sets of secrets, user managed secrets & others which are needed for server operations. Each of server managed secrets should be prefixed by _OPTIMUS_<key name> and will not be allowed to be used by users in the job spec. Optimus should also disallow anyone using this prefix to register their secrets. The secrets can be namespaced by optimus namespace or at project level, and will be accessible accordingly. Secret names should be maintained unique across a project.

    All secret names within Optimus systems are considered case insensitive, treating different letter cases as identical. Furthermore, to promote consistency and ease of management, all secret names are uniformly stored in uppercase letters, regardless of their original format. The implementation of this policy is essential to enhance security, minimize potential inconsistencies, and facilitate seamless secret retrieval and utilization.

    Using secrets​

    Secrets can be used as part of the job spec config using macros with their names. This will work as aliasing the secret to be used in containers. Only the secrets created at project & namespace the job belongs to can be referenced. So, for the plugin writers any secret that plugin needs can be accessed through environment variables defined in the job spec or can get the secrets by defining in any assets.

    task: foo
    config:
      do: this
      dsn: {{ .secret.postgres_dsn }}

    One thing to note is currently we print all the container environment variables using printenv command as debug. This should be removed after this RFC is merged to avoid exposing secrets in container logs.

    Only the admins & containers to be authorized for registerinstance end point, as this will allow access to all secrets.

    Because Optimus is deployed in trusted network, we don't need TLS for now to fetch job secrets but once Optimus is deployed as a service on edge network, this communication should only happen over TLS.

    Authentication & Authorization​

    Even though Optimus doesn't have its own authentication, expect users to bring in their own auth proxy infront of Optimus. All user access & container access will be restricted through the auth proxy. The corresponding secret through which the containers running in the kubernetes cluster will be authenticated need to be precreated per project.

    Optimus CLI​

    User interaction to manage a secret will start from CLI. Users can create/update/list/delete a secret as follows

    By default secrets will be created under their namespace, but optionally the secret can be created at project level by not providing any namespace while creation. This is needed if users want to allow access across entire project.

    Secrets can be accessed by providing the project & namespace the secret is created in, if the secret is created at project level then namespace can be set to empty string if optimus.yaml already has the namespace configured.

    Create/Update​

    optimus secret create/update <name> <value> will take a secret name and value

    optimus secret create/update <name> --file="path" should read the file content as value.

    Additional flag --base64 can be provided by user stating the value is already encoded, if not provided optimus ensures to encode & store it, basic checks can be done to check if the string is a valid base64 encoded string.

    Delete​

    optimus secret delete <name>

    List​

    optimus secret list to list all created secrets in a project/namespace, along with the creation/updated time, will be helpful such that users can use in the job spec, as users might forget the key name, this will not list the system managed secrets.

    List operation will print a digest of the secret. Digest should be a SHA hash of the encrypted string to simply visualize it as a signature when a secret is changed or the key gets rotated.

    Example:

         NAME     |              DIGEST              |  NAMESPACE |  DATE
      SECRET_1    | 6c463e806738046ff3c78a08d8bd2b70 |     *      | 2021-10-06 02:02:02
      SECRET_2    | 3aa788a21a76651c349ceeee76f1cb76 |   finance  | 2021-10-06 06:02:02
      SECRET_2    | 3aa788a21a76651c349ceeee76f1cb76 |  transport | 2021-10-06 06:02:02

    This command will only shows the user managed secret sets and ignoring the system managed secret, while on the REST response both sets can be shown. An additional field in secret table called 'TYPE' can be added to differentiate the two sets.

    Using secrets without Optimus​

    If someone wants to pass an exclusive secret without registering it with Optimus first, that should also be possible.

    • In case of k8s: this can be done using a new field introduced in Job spec as metadata which will allow users to mount arbitrary secrets inside the container available in the same k8s namespace.

    Rotating Optimus Server key​

    There is a need for rotating Optimus Server Key when it is compromised. As the server key is configured through environment variable, the rotation can happen by configuring through environment variables. There can be two environment variables for server keys OLD_APP_KEY & APP_KEY. During startup sha of the OLD_APP_KEY is compared with the sha stored in the database, if it matches then rotation will happen and at the end of rotation the sha will be replaced with APP_KEY's sha. The comparision is needed to check to not attempt rotation during restarts. If there are multiple replicas then as we do this in a transaction only one succeeds.

    This step will internally loading all the secrets that belong to a project to memory, decrypting it with the old_key, and encrypting it with the new key, the entire operation will happen in a single db transaction.

    Migration​

    • This design will be a breaking change compare to how the secrets are handled and will require all the current secrets to be registered again. -Current system managed secrets will be re-registered using _OPTIMUS_ prefix. Plugin secrets will also need to be registered to Optimus.

    Footnotes & References

    - +Current system managed secrets will be re-registered using _OPTIMUS_ prefix. Plugin secrets will also need to be registered to Optimus.

    Footnotes & References

    + \ No newline at end of file diff --git a/docs/rfcs/simplify_plugin_maintenance/index.html b/docs/rfcs/simplify_plugin_maintenance/index.html index efd470a4ae..8be869324d 100644 --- a/docs/rfcs/simplify_plugin_maintenance/index.html +++ b/docs/rfcs/simplify_plugin_maintenance/index.html @@ -10,13 +10,13 @@ - +
    -

    simplify_plugin_maintenance

    • Feature Name: Simplify Plugins
    • Status: Draft
    • Start Date: 2022-05-07
    • Author: Saikumar

    Summary

    The scope of this rfc is to simplify the release and deployment operations w.r.t the optimus plugin ecosystem.

    The proposal here is to :

    1. Avoid Wrapping Executor Images :
      Decouple the executor_boot_process and the executor as separate containers where the airflow worker launches a pod with init-container (for boot process) adjacent to executor container.
    2. Simplfy Plugin Installation :
      • Server end : Install plugins declaratively at runtime instead of manually baking them into the optimus server image (in kubernetes setup).
      • Client end : Plugin interface for client-end is limited to support Version, Survey Questions and Answers etc. that can be extracted out from the current plugin interface and maintained as yaml file which simplifies platform dependent plugin distribution for cli.

    Technical Design

    Background :​

    Changes that trigger a new release in Optimus setup:​

    • Executor Image changes
    • Executor Image Wrapper changes
    • Plugin binary changes
    • Optimus binary changes

    Release dependencies as per current design​

    • Executor Image release -> Executor Wrapper Image release -> Plugin binary release -> Server release
    • Plugin binary release -> Server release

    1. Avoid Wrapping Executor Images :​

    • The executor_boot_process and executor are coupled:
    -- Plugin repo structure
    /task/
    /task/Dockerfile           -- task_image
    /task/executor/Dockerfile  -- executor_image

    Executor Wrapper Image (Task Image) :

    • It's a wrapper around the executor_image to facilitate boot mechanism for executor.
    • The optimus binary is downloaded during buildtime of this image.
    • During runtime, it does as follow :
      • Fetch assets, secrets, env from optimus server.
      • Load the env and launches the executor process.
    task_image 
        | executor_image
        | optimus-bin
        | entrypoint.sh (load assets, env and launch executor)

    The optimus-bin and entrypoint.sh are baked into the task_image and is being maintained by task/plugin developers.

    2. Simplify Plugin Installation :​

    • Plugin binaries are manually installed (baked into optimus image in kubernetes setup).
    • Any change in plugin code demands re-creation of optimus image with new plugin binary, inturn demanding redeployment of optimus server. (in kubernetes setup)
    • At client side, plugin binaries require support for different platforms.

    Approach :​

    1. Avoid Wrapping Executor Images :​

    • Decouple the lifecycle of the executor and the boot process as seperate containers/images.
    Simplify Plugins Executor

    Task Boot Sequence:

    1. Airflow worker fetches env and secrets for the job and adds them to the executor pod as environment variables.
    2. KubernetesPodOperator spawns init-container and executor-container, mounted with shared volume (type emptyDir) for assets.
    3. init-container fetches assets, config, env files and writes onto the shared volume.
    4. the default entrypoint in the executor-image starts the actual job.

    2. Simplify Plugin Installations :​

    A) Plugin Manager:​

    • Currently the plugins are maintained as monorepo and versioned together. For any change in a single plugin, a new tar file containing all plugin binaries is created and released.
    • A plugin manager is required to support declarative installation of plugins so that plugins can be independently versioned, packaged and installed.
    • This plugin manager consumes a config (plugins_config) and downloads artifacts from a plugin repository.
    • Optimus support for plugin manager as below.
      • optimus plugin install -c config.yaml -- at server
    • Support for different kinds of plugin repositories (like s3, gcs, url, local file system etc..) gives the added flexibility and options to distribute and install the plugin binaries in different ways.
    • Plugins are installed at container runtime and this decouples the building of optimus docker image from plugins installations.
    • Example for the plugin_config: 
    plugin:
      dir: .plugins
      artifacts:
        # local filesystem for dev
        - ../transformers/dist/bq2bq_darwin_arm64/optimus-bq2bq_darwin_arm64
        # any http uri
        - https://github.com/goto/optimus/releases/download/v0.2.5/optimus_0.2.5_linux_arm64.tar.gz

    B) Yaml Plugin Interface: (for client side simplification)​

    • Currently plugins are implemented and distributed as binaries and as clients needs to install them, it demands support for different host architectures.
    • Since CLI (client side) plugins just require details about plugin such as Version, Suevery Questions etc. the proposal here is to maintain CLI plugins as yaml files.
    • Implementation wise, the proposal here is to split the current plugin interface (which only supports interaction with binary plugins) to also accommodate yaml based plugins.
    • The above mentioned pluign manager, at server end, would be agnostic about the contents of plugin artifacts from the repository.
    • At client side, the CLI could sync the yaml files from the server to stay up-to-date with the server w.r.t plugins.
    • At this point, we have the scope to move away from binary plugins except for bq2bq plugin due to its depdendency on ComplileAsset and ResolveDependency functionalities which are required at server end (not at cli).
    • Handling Bq2Bq plugin:
      • Move CompileAsset functionality as a part of Bq2bq executor.
      • Move ResolveDependency functionality to optimus core which should support dependecy-resolution on standard-sql
    • Meanwhile the Bq2bq plugin is handled, the plugin interface can be maintanined in such a way that it also supports binary plugin in addition to yaml (as backward compaitbility feature).
    • The plugin discovery logic should be to load binary if present, else load yaml file; for a single plugin.
    • Now that we have yaml files at server, CLI can sync only the yaml files from the server.
      • optimus plugin sync -c optimus.yaml
    • Example representation of the yaml plugin : 
    name: bq2bq
    description: BigQuery to BigQuery transformation task
    plugintype: task
    pluginmods:
      - cli
      - dependencyresolver
    pluginversion: 0.1.0-SNAPSHOT-27cb56f
    apiversion: []
    image: docker.io/goto/optimus-task-bq2bq-executor:0.1.0-SNAPSHOT-27cb56f
    secretpath: /tmp/auth.json
    dependson: []
    hooktype: ""

    questions:
      - name: PROJECT
        prompt: Project ID
        help: Destination bigquery project ID
        regexp: ^[a-zA-Z0-9_\-]+$
        validationerror: invalid name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)
        minlength: 3
      - name: Dataset
        prompt: Dataset Name
        help: Destination bigquery dataset ID
        regexp: ^[a-zA-Z0-9_\-]+$
        validationerror: invalid name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)
        minlength: 3
      - name: TABLE
        prompt: Table ID
        help: Destination bigquery table ID
        regexp: ^[a-zA-Z0-9_-]+$
        validationerror: invalid table name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)
        minlength: 3
        maxlength: 1024
      - name: LOAD_METHOD
        prompt: Load method to use on destination
        help: |
          APPEND        - Append to existing table
          REPLACE       - Deletes existing partition and insert result of select query
          MERGE         - DML statements, BQ scripts
          REPLACE_MERGE - [Experimental] Advanced replace using merge query
        default: APPEND
        multiselect:
          - APPEND
          - REPLACE
          - MERGE
          - REPLACE_MERGE
          - REPLACE_ALL
    defaultassets:
      - name: query.sql
        value: |
          -- SQL query goes here

          Select * from "project.dataset.table";

    Result:​

    Simplify Plugins
    • Executor boot process is standardised and extracted away from plugin developers. Now any arbitrary image can be used for executors.
    • At server side, for changes in plugin (dur to plugin release), update the plugin_manager_config and restart the optimus server pod. The plugin manager is expected to reinstall the plugins.
    • Client side dependency on plugins is simplified with yaml based plugins.
    - +

    simplify_plugin_maintenance

    • Feature Name: Simplify Plugins
    • Status: Draft
    • Start Date: 2022-05-07
    • Author: Saikumar

    Summary

    The scope of this rfc is to simplify the release and deployment operations w.r.t the optimus plugin ecosystem.

    The proposal here is to :

    1. Avoid Wrapping Executor Images :
      Decouple the executor_boot_process and the executor as separate containers where the airflow worker launches a pod with init-container (for boot process) adjacent to executor container.
    2. Simplfy Plugin Installation :
      • Server end : Install plugins declaratively at runtime instead of manually baking them into the optimus server image (in kubernetes setup).
      • Client end : Plugin interface for client-end is limited to support Version, Survey Questions and Answers etc. that can be extracted out from the current plugin interface and maintained as yaml file which simplifies platform dependent plugin distribution for cli.

    Technical Design

    Background :​

    Changes that trigger a new release in Optimus setup:​

    • Executor Image changes
    • Executor Image Wrapper changes
    • Plugin binary changes
    • Optimus binary changes

    Release dependencies as per current design​

    • Executor Image release -> Executor Wrapper Image release -> Plugin binary release -> Server release
    • Plugin binary release -> Server release

    1. Avoid Wrapping Executor Images :​

    • The executor_boot_process and executor are coupled:
    -- Plugin repo structure
    /task/
    /task/Dockerfile           -- task_image
    /task/executor/Dockerfile  -- executor_image

    Executor Wrapper Image (Task Image) :

    • It's a wrapper around the executor_image to facilitate boot mechanism for executor.
    • The optimus binary is downloaded during buildtime of this image.
    • During runtime, it does as follow :
      • Fetch assets, secrets, env from optimus server.
      • Load the env and launches the executor process.
    task_image 
        | executor_image
        | optimus-bin
        | entrypoint.sh (load assets, env and launch executor)

    The optimus-bin and entrypoint.sh are baked into the task_image and is being maintained by task/plugin developers.

    2. Simplify Plugin Installation :​

    • Plugin binaries are manually installed (baked into optimus image in kubernetes setup).
    • Any change in plugin code demands re-creation of optimus image with new plugin binary, inturn demanding redeployment of optimus server. (in kubernetes setup)
    • At client side, plugin binaries require support for different platforms.

    Approach :​

    1. Avoid Wrapping Executor Images :​

    • Decouple the lifecycle of the executor and the boot process as seperate containers/images.
    Simplify Plugins Executor

    Task Boot Sequence:

    1. Airflow worker fetches env and secrets for the job and adds them to the executor pod as environment variables.
    2. KubernetesPodOperator spawns init-container and executor-container, mounted with shared volume (type emptyDir) for assets.
    3. init-container fetches assets, config, env files and writes onto the shared volume.
    4. the default entrypoint in the executor-image starts the actual job.

    2. Simplify Plugin Installations :​

    A) Plugin Manager:​

    • Currently the plugins are maintained as monorepo and versioned together. For any change in a single plugin, a new tar file containing all plugin binaries is created and released.
    • A plugin manager is required to support declarative installation of plugins so that plugins can be independently versioned, packaged and installed.
    • This plugin manager consumes a config (plugins_config) and downloads artifacts from a plugin repository.
    • Optimus support for plugin manager as below.
      • optimus plugin install -c config.yaml -- at server
    • Support for different kinds of plugin repositories (like s3, gcs, url, local file system etc..) gives the added flexibility and options to distribute and install the plugin binaries in different ways.
    • Plugins are installed at container runtime and this decouples the building of optimus docker image from plugins installations.
    • Example for the plugin_config: 
    plugin:
      dir: .plugins
      artifacts:
        # local filesystem for dev
        - ../transformers/dist/bq2bq_darwin_arm64/optimus-bq2bq_darwin_arm64
        # any http uri
        - https://github.com/goto/optimus/releases/download/v0.2.5/optimus_0.2.5_linux_arm64.tar.gz

    B) Yaml Plugin Interface: (for client side simplification)​

    • Currently plugins are implemented and distributed as binaries and as clients needs to install them, it demands support for different host architectures.
    • Since CLI (client side) plugins just require details about plugin such as Version, Suevery Questions etc. the proposal here is to maintain CLI plugins as yaml files.
    • Implementation wise, the proposal here is to split the current plugin interface (which only supports interaction with binary plugins) to also accommodate yaml based plugins.
    • The above mentioned pluign manager, at server end, would be agnostic about the contents of plugin artifacts from the repository.
    • At client side, the CLI could sync the yaml files from the server to stay up-to-date with the server w.r.t plugins.
    • At this point, we have the scope to move away from binary plugins except for bq2bq plugin due to its depdendency on ComplileAsset and ResolveDependency functionalities which are required at server end (not at cli).
    • Handling Bq2Bq plugin:
      • Move CompileAsset functionality as a part of Bq2bq executor.
      • Move ResolveDependency functionality to optimus core which should support dependecy-resolution on standard-sql
    • Meanwhile the Bq2bq plugin is handled, the plugin interface can be maintanined in such a way that it also supports binary plugin in addition to yaml (as backward compaitbility feature).
    • The plugin discovery logic should be to load binary if present, else load yaml file; for a single plugin.
    • Now that we have yaml files at server, CLI can sync only the yaml files from the server.
      • optimus plugin sync -c optimus.yaml
    • Example representation of the yaml plugin : 
    name: bq2bq
    description: BigQuery to BigQuery transformation task
    plugintype: task
    pluginmods:
      - cli
      - dependencyresolver
    pluginversion: 0.1.0-SNAPSHOT-27cb56f
    apiversion: []
    image: docker.io/goto/optimus-task-bq2bq-executor:0.1.0-SNAPSHOT-27cb56f
    secretpath: /tmp/auth.json
    dependson: []
    hooktype: ""

    questions:
      - name: PROJECT
        prompt: Project ID
        help: Destination bigquery project ID
        regexp: ^[a-zA-Z0-9_\-]+$
        validationerror: invalid name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)
        minlength: 3
      - name: Dataset
        prompt: Dataset Name
        help: Destination bigquery dataset ID
        regexp: ^[a-zA-Z0-9_\-]+$
        validationerror: invalid name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)
        minlength: 3
      - name: TABLE
        prompt: Table ID
        help: Destination bigquery table ID
        regexp: ^[a-zA-Z0-9_-]+$
        validationerror: invalid table name (can only contain characters A-Z (in either case), 0-9, hyphen(-) or underscore (_)
        minlength: 3
        maxlength: 1024
      - name: LOAD_METHOD
        prompt: Load method to use on destination
        help: |
          APPEND        - Append to existing table
          REPLACE       - Deletes existing partition and insert result of select query
          MERGE         - DML statements, BQ scripts
          REPLACE_MERGE - [Experimental] Advanced replace using merge query
        default: APPEND
        multiselect:
          - APPEND
          - REPLACE
          - MERGE
          - REPLACE_MERGE
          - REPLACE_ALL
    defaultassets:
      - name: query.sql
        value: |
          -- SQL query goes here

          Select * from "project.dataset.table";

    Result:​

    Simplify Plugins
    • Executor boot process is standardised and extracted away from plugin developers. Now any arbitrary image can be used for executors.
    • At server side, for changes in plugin (dur to plugin release), update the plugin_manager_config and restart the optimus server pod. The plugin manager is expected to reinstall the plugins.
    • Client side dependency on plugins is simplified with yaml based plugins.
    + \ No newline at end of file diff --git a/docs/rfcs/support_for_depending_on_external_sensors/index.html b/docs/rfcs/support_for_depending_on_external_sensors/index.html index b0c14d4eb3..446879bbd6 100644 --- a/docs/rfcs/support_for_depending_on_external_sensors/index.html +++ b/docs/rfcs/support_for_depending_on_external_sensors/index.html @@ -10,13 +10,13 @@ - +
    -

    support_for_depending_on_external_sensors

    • Feature Name: Support For Depndening on External Sources
    • Status: Draft
    • Start Date: 2022-01-23
    • Authors:

    Summary

    Optimus supports job dependencies, but there is a need for optimus jobs to depend on external sources which are not managed by the optimus server. For example, depending the BQ or GCS data availability or data being managed by another optimus server. Whatever data sources optimus is managing lets have sensors for basic data availability check, in GCS checking for file exists & in BQ taking a select query & returning success when rowcount > 0. For other requirements let's have a http sensor.

    Technical Design

    Optimus can add support for all the sensors as libraries, which will be evaulated within the execution envrionment of the user, all variables will be returned for a given scheduled date through the api call which will be used by the actual sensor execution.

    Optimus provides libraries needed for the above operations which can be used in the respective execution environment of the scheduler, currently the library will be offered in python.

    The /intance api call can accept params to filter what to return to just reduce the unnecessary payload & return only the needed variables, as sensors execute a lot.

    Http Sensor​

    If the call returns 200 then the sensor succeeds

    dependencies : 
     type : http
     endpoint : url
     headers :
     body :

    BQ Sensor​

    If the query results in rows then the sensor succeeds

    dependencies : 
     type : bq
     query : 
     service_account :

    GCS Sensor​

    If the path exists then the sensor succeeds

    dependencies : 
     type : gcs
     path : 
     service_account :
    - +

    support_for_depending_on_external_sensors

    • Feature Name: Support For Depndening on External Sources
    • Status: Draft
    • Start Date: 2022-01-23
    • Authors:

    Summary

    Optimus supports job dependencies, but there is a need for optimus jobs to depend on external sources which are not managed by the optimus server. For example, depending the BQ or GCS data availability or data being managed by another optimus server. Whatever data sources optimus is managing lets have sensors for basic data availability check, in GCS checking for file exists & in BQ taking a select query & returning success when rowcount > 0. For other requirements let's have a http sensor.

    Technical Design

    Optimus can add support for all the sensors as libraries, which will be evaulated within the execution envrionment of the user, all variables will be returned for a given scheduled date through the api call which will be used by the actual sensor execution.

    Optimus provides libraries needed for the above operations which can be used in the respective execution environment of the scheduler, currently the library will be offered in python.

    The /intance api call can accept params to filter what to return to just reduce the unnecessary payload & return only the needed variables, as sensors execute a lot.

    Http Sensor​

    If the call returns 200 then the sensor succeeds

    dependencies : 
     type : http
     endpoint : url
     headers :
     body :

    BQ Sensor​

    If the query results in rows then the sensor succeeds

    dependencies : 
     type : bq
     query : 
     service_account :

    GCS Sensor​

    If the path exists then the sensor succeeds

    dependencies : 
     type : gcs
     path : 
     service_account :
    + \ No newline at end of file diff --git a/docs/rfcs/template/index.html b/docs/rfcs/template/index.html index c613d7fa1a..1c780ceae4 100644 --- a/docs/rfcs/template/index.html +++ b/docs/rfcs/template/index.html @@ -10,13 +10,13 @@ - +
    -

    template

    • Feature Name:
    • Status: draft/in-progress/completed/rejected/obsolete/postponed
    • Start Date: YYYY-MM-DD
    • Authors:

    Remember, you can submit a PR with your RFC before the text is complete.

    Summary

    • What is being proposed
    • Why (short reason)
    • How (short plan)
    • Impact

    Audience: all participants

    Technical design

    Audience: project members, expert users.

    Drawbacks​

    ...

    Rationale and Alternatives​

    ...

    Unresolved questions

    Audience: all participants

    - +

    template

    • Feature Name:
    • Status: draft/in-progress/completed/rejected/obsolete/postponed
    • Start Date: YYYY-MM-DD
    • Authors:

    Remember, you can submit a PR with your RFC before the text is complete.

    Summary

    • What is being proposed
    • Why (short reason)
    • How (short plan)
    • Impact

    Audience: all participants

    Technical design

    Audience: project members, expert users.

    Drawbacks​

    ...

    Rationale and Alternatives​

    ...

    Unresolved questions

    Audience: all participants

    + \ No newline at end of file diff --git a/docs/server-guide/configuration/index.html b/docs/server-guide/configuration/index.html index d248c8b651..3f56833741 100644 --- a/docs/server-guide/configuration/index.html +++ b/docs/server-guide/configuration/index.html @@ -10,13 +10,13 @@ - +
    -

    Server Configuration

    See the server configuration example on config.sample.yaml.

    ConfigurationDescription
    LogLogging level & format configuration.
    ServeRepresents any configuration needed to start Optimus, such as port, host, DB details, and application key (for secrets encryption).
    TelemetryCan be used for tracking and debugging using Jaeger.
    PluginOptimus will try to look for the plugin artifacts through this configuration.
    Resource ManagerIf your server has jobs that are dependent on other jobs in another server, you can add that external Optimus server host as a resource manager.

    Note:

    Application key can be randomly generated using: 

    head -c 50 /dev/random | base64

    Just take the first 32 characters of the string.

    - +

    Server Configuration

    See the server configuration example on config.sample.yaml.

    ConfigurationDescription
    LogLogging level & format configuration.
    ServeRepresents any configuration needed to start Optimus, such as port, host, DB details, and application key (for secrets encryption).
    TelemetryCan be used for tracking and debugging using Jaeger.
    PluginOptimus will try to look for the plugin artifacts through this configuration.
    Resource ManagerIf your server has jobs that are dependent on other jobs in another server, you can add that external Optimus server host as a resource manager.

    Note:

    Application key can be randomly generated using: 

    head -c 50 /dev/random | base64

    Just take the first 32 characters of the string.

    + \ No newline at end of file diff --git a/docs/server-guide/db-migrations/index.html b/docs/server-guide/db-migrations/index.html index 4ec11fbc2e..f756e140fb 100644 --- a/docs/server-guide/db-migrations/index.html +++ b/docs/server-guide/db-migrations/index.html @@ -10,15 +10,15 @@ - +

    DB Migrations

    Migrate to Specific Version​

    Upgrade the DB to the specified migration version.

    $ optimus migration to --version [version]

    Note: To migrate to the latest one, running the Optimus’ serve command should be enough. It will migrate to the latest automatically.

    Rollback​

    Revert the current active migration to several previous migration versions.

    $ optimus migration rollback --count [n]

    [n] is the number of migrations to rollback.

    Export & Upload​

    In some cases, due to differences in how Optimus is storing the job and resource specifications in DB in 1 version to -another version, you might want to export the jobs in the server to your local YAML files and redeploy it.

    The export is possible using the following command:

    $ optimus job export –-dir job_result

    This means, you will export all of the jobs from all of the projects in your Optimus server to the job_result directory. It is also possible to run this export command against a single project/namespace/job.

    For exporting resources, use the following command:

    $ optimus resource export -dir resource_result

    Similar to the job command, it is also possible to export resources only for a single project/namespace/resource.

    - +another version, you might want to export the jobs in the server to your local YAML files and redeploy it.

    The export is possible using the following command:

    $ optimus job export –-dir job_result

    This means, you will export all of the jobs from all of the projects in your Optimus server to the job_result directory. It is also possible to run this export command against a single project/namespace/job.

    For exporting resources, use the following command:

    $ optimus resource export -dir resource_result

    Similar to the job command, it is also possible to export resources only for a single project/namespace/resource.

    + \ No newline at end of file diff --git a/docs/server-guide/installing-plugins/index.html b/docs/server-guide/installing-plugins/index.html index 16c780a271..c7f11a7d38 100644 --- a/docs/server-guide/installing-plugins/index.html +++ b/docs/server-guide/installing-plugins/index.html @@ -10,15 +10,15 @@ - +

    Installing Plugins

    Plugin needs to be installed in the Optimus server before it can be used. Optimus uses the following directories for discovering plugin binaries

    ./.plugins
    ./
    <exec>/
    <exec>/.optimus/plugins
    $HOME/.optimus/plugins
    /usr/bin
    /usr/local/bin

    Even though the above list of directories is involved in plugin discovery, it is advised to use .plugins in the -current working directory of the project or Optimus binary.

    To simplify installation, you can add plugin artifacts in the server config:

    plugin:
      artifacts:
       - https://...path/to/optimus-plugin-neo.yaml  # http
       - http://.../plugins.zip # zip
       - ../transformers/optimus-bq2bq_darwin_arm64 # relative paths
       - ../transformers/optimus-plugin-neo.yaml

    Run below command to auto-install the plugins in the .plugins directory.

    $ optimus plugin install -c config.yaml  # This will install plugins in the `.plugins` folder.
    - +current working directory of the project or Optimus binary.

    To simplify installation, you can add plugin artifacts in the server config:

    plugin:
      artifacts:
       - https://...path/to/optimus-plugin-neo.yaml  # http
       - http://.../plugins.zip # zip
       - ../transformers/optimus-bq2bq_darwin_arm64 # relative paths
       - ../transformers/optimus-plugin-neo.yaml

    Run below command to auto-install the plugins in the .plugins directory.

    $ optimus plugin install -c config.yaml  # This will install plugins in the `.plugins` folder.
    + \ No newline at end of file diff --git a/docs/server-guide/starting-optimus-server/index.html b/docs/server-guide/starting-optimus-server/index.html index 61f6e4f0fa..59442294a3 100644 --- a/docs/server-guide/starting-optimus-server/index.html +++ b/docs/server-guide/starting-optimus-server/index.html @@ -10,7 +10,7 @@ - + @@ -19,8 +19,8 @@ variable OPTIMUS_[CONFIGNAME], or config.yaml file in Optimus binary directory.

    1. Using --config flag

    $ optimus serve --config /path/to/config/file.yaml

    If you specify the configuration file using the --config flag, then any configs defined in the env variable and default config.yaml from the Optimus binary directory will not be loaded.

    2. Using environment variable

    All the configs can be passed as environment variables using OPTIMUS_[CONFIG_NAME] convention. [CONFIG_NAME] is the key name of config using _ as the path delimiter to concatenate between keys.

    For example, to use the environment variable, assuming the following configuration layout:

    version: 1
    serve:
      port: 9100
      app_key: randomhash

    Here is the corresponding environment variable for the above

    Configuration keyEnvironment variable
    versionOPTIMUS_VERSION
    serve.portOPTIMUS_PORT
    serve.app_keyOPTIMUS_SERVE_APP_KEY

    Set the env variable using export

    $ export OPTIMUS_PORT=9100

    Note: If you specify the env variable and you also have config.yaml in the Optimus binary directory, then any configs -from the env variable will override the configs defined in config.yaml in Optimus binary directory.

    3. Using default config.yaml from Optimus binary directory

    $ which optimus
    /usr/local/bin/optimus

    So the config.yaml file can be loaded on /usr/local/bin/config.yaml

    - +from the env variable will override the configs defined in config.yaml in Optimus binary directory.

    3. Using default config.yaml from Optimus binary directory

    $ which optimus
    /usr/local/bin/optimus

    So the config.yaml file can be loaded on /usr/local/bin/config.yaml

    + \ No newline at end of file diff --git a/help/index.html b/help/index.html index 13cf391bf2..a7b182bad9 100644 --- a/help/index.html +++ b/help/index.html @@ -10,13 +10,13 @@ - +

    Need help?

    Need a bit of help? We're here for you. Check out our current issues, GitHub discussions, or get support through Slack.

    Slack

    The Optimus team has an open source slack workspace to discuss development and support. Most of the Optimus discussions happen in #optimus channel.
    Join us on Slack

    GitHub Issues

    Have a general issue or bug that you've found? We'd love to hear about it in our GitHub issues. This can be feature requests too!
    Go to issues

    GitHub Discussions

    For help and questions about best practices, join our GitHub discussions. Browse and ask questions.
    Go to discussions
    - + \ No newline at end of file diff --git a/index.html b/index.html index 8424f2de0b..f0e5a16d2f 100644 --- a/index.html +++ b/index.html @@ -10,13 +10,13 @@ - +

    Built for scale

    Optimus is an easy-to-use, reliable, and performant workflow orchestrator for data transformation, data modeling, pipelines, and data quality management. It enables data analysts and engineers to transform their data by writing simple SQL queries and YAML configuration while Optimus handles dependency management, scheduling and all other aspects of running transformation jobs at scale.

    Zero dependency

    Optimus is written in Go and compiles into a single binary with no external dependencies, and requires a very minimal memory footprint.

    Warehouse management

    Optimus allows you to create and manage your data warehouse tables and views through YAML based configuration.

    Extensible

    With the ease of plugin development build your own plugins. Optimus support Python transformation and allows for writing custom plugins.

    CLI

    Optimus comes with a CLI which allows you to interact with workflows effectively. You can create, run, replay jobs and more.

    Proven

    Battle tested at large scale across multiple companies. Largest deployment runs thousands of workflows on multiple data sources.

    Workflows

    Optimus provides industry-proven workflows using git and REST/GRPC based specification management for data warehouse management.

    Key features

    Optimus is an ETL orchestration tool that helps manage warehouse resources and schedule transformation over cron interval. Warehouses like Bigquery can be used to create, update, read, delete different types of resources(dataset/table/standard view). Similarly, jobs can be SQL transformations taking inputs from single/multiple source tables executing over fixed schedule interval. Optimus was made from start to be extensible, which is, adding support of different kind of warehouses, transformers can be done easily.

    Scheduling

    Optimus provides an easy way to schedule your SQL transformation through a YAML based configuration.

    Dependency resolution

    Optimus parses your data transformation queries and builds a dependency graphs automaticaly instead of users defining it in DAGs.

    Dry runs

    Before SQL query is scheduled for transformation, during deployment query will be dry-run to make sure it passes basic sanity checks.

    Powerful templating

    Optimus provides query compile time templating with variables, loop, if statements, macros, etc for allowing users to write complex tranformation logic.

    Cross tenant dependency

    Optimus is a multi-tenant service. With more than two tenants registered Optimus can resolve cross tenant dependencies automatically.

    Hooks

    Optimus provides hooks for post tranformation logic to extend the functionality of the transformation. e,g. You can sink BigQuery tables to Kafka.

    Workflow

    With Optimus data teams work directly with the data warehouse and data catalogs. Optimus provides a set of workflows which can be used to build data transformation pipelines, reporting, operational, machine learning workflows.

    Develop

    Write your specifications in git using Optimus CLI or use Optimus APIs to prgramtically submit specifications through SDK.

    Test

    Test your workflows prior to production with linting, dry runs and local execution from your machines.

    Deploy

    Deploy your workflows safely with Optimus CLI and APIs to production.
    - + \ No newline at end of file