From 96bcd653e19847d795be9b387741a239c95db822 Mon Sep 17 00:00:00 2001 From: Jon Magoon Date: Wed, 30 Oct 2024 13:17:53 -0600 Subject: [PATCH 1/4] Rename ServerError.ts to serverError.ts (#238) --- frontend/server/{ServerError.ts => serverError.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename frontend/server/{ServerError.ts => serverError.ts} (100%) diff --git a/frontend/server/ServerError.ts b/frontend/server/serverError.ts similarity index 100% rename from frontend/server/ServerError.ts rename to frontend/server/serverError.ts From b8fb6df0d4c67175d4b5217a4a3d22156ae76e77 Mon Sep 17 00:00:00 2001 From: plparent <45149203+plparent@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:24:22 -0400 Subject: [PATCH 2/4] Local gpu (k3d) (#240) * Adding k3s and k3d as drivers * k3d fixes * bugfixes * wip --- executor.go | 13 +++++++++++++ main.go | 2 ++ setup.go | 2 +- translator.go | 12 +++++++++++- 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/executor.go b/executor.go index bffcc714..dbf11315 100644 --- a/executor.go +++ b/executor.go @@ -705,6 +705,19 @@ func localExecute(pipeline *zjson.Pipeline, pipelineMerkleTree *zjson.PipelineMe } } + if cfg.Local.Driver == "k3d" { + imageImport := []string{"image", "import"} + for _, image := range blocks { + imageImport = append(imageImport, image) + } + imageImport = append(imageImport, "-c") + imageImport = append(imageImport, cfg.Local.K3DCluster) + pullImage := cmd.NewCmd("k3d", imageImport...) + <-pullImage.Start() + log.Println(pullImage.Status().Stdout) + log.Println(pullImage.Status().Stderr) + } + if err := eg.Wait(); err != nil { pipelineLogger.Printf("error during pipeline build execution; err=%v", err) log.Printf("error during pipeline build execution; err=%v", err) diff --git a/main.go b/main.go index 0226a222..d6ad25d4 100644 --- a/main.go +++ b/main.go @@ -52,9 +52,11 @@ type Config struct { Cloud Cloud `json:"Cloud,omitempty"` } + type Local struct { BucketPort int Driver string + K3DCluster string `json:"K3DCluster,omitempty"` } type Cloud struct { diff --git a/setup.go b/setup.go index 6f1d73e7..269840dc 100644 --- a/setup.go +++ b/setup.go @@ -308,7 +308,7 @@ func setup(ctx context.Context, config Config, db *sql.DB) { signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt) - if config.Local.Driver == "minikube" { + if config.Local.Driver == "minikube" || config.Local.Driver == "k3d" { stopBucketCh := make(chan struct{}, 1) go func() { for { diff --git a/translator.go b/translator.go index feac5f99..bb14d3d5 100644 --- a/translator.go +++ b/translator.go @@ -52,7 +52,17 @@ func checkImage(ctx context.Context, image string, cfg Config) (bool, bool, erro } } return false, false, nil - } else { + } else if cfg.Local.Driver == "k3d" { + dockerImage := cmd.NewCmd("docker", "exec", "k3d-" + cfg.Local.K3DCluster + "-server-0", "crictl", "images") + <-dockerImage.Start() + for _, line := range dockerImage.Status().Stdout { + data := strings.Fields(line) + if "docker.io/zetaforge/"+image == data[0] + ":" + data[1] { + return true, false, nil + } + } + return false, false, nil + } else { apiClient, err := client.NewClientWithOpts( client.WithAPIVersionNegotiation(), ) From 92f26b5245be82a567f07651b04bbcc9e072dec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20LaBerge?= <35940434+FGRCL@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:30:04 -0400 Subject: [PATCH 3/4] Code editor delete bug (#229) * wip, try to update the CodeMirror compoenent only on file load * fix some syncing issues * use ref to avoid rerendering when the code editor changes * remove unused component * add package * Manually update the code mirror compoenent and keep the cursor position * Extract to function --- frontend/package-lock.json | 8 +-- frontend/package.json | 2 +- .../directoryViewer/CodeEditor.jsx | 9 ++- .../directoryViewer/CodeManualEditor.jsx | 33 ++++++++++- .../directoryViewer/CodeMirrorComponents.jsx | 56 ++++++++++--------- frontend/src/hooks/useConfirmModal.jsx | 6 +- frontend/src/hooks/useFileBuffer.jsx | 17 +++--- frontend/src/hooks/useFileHandle.jsx | 2 +- 8 files changed, 86 insertions(+), 47 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b1505017..19bb487d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -26,7 +26,7 @@ "rfdc": "^1.3.1", "s3-sync-client": "^4.3.1", "source-map-support": "^0.5.21", - "use-debounce": "^10.0.3", + "use-debounce": "^10.0.4", "use-immer": "^0.9.0", "uuidv7": "^0.6.3" }, @@ -14400,9 +14400,9 @@ } }, "node_modules/use-debounce": { - "version": "10.0.3", - "resolved": "https://registry.npmjs.org/use-debounce/-/use-debounce-10.0.3.tgz", - "integrity": "sha512-DxQSI9ZKso689WM1mjgGU3ozcxU1TJElBJ3X6S4SMzMNcm2lVH0AHmyXB+K7ewjz2BSUKJTDqTcwtSMRfB89dg==", + "version": "10.0.4", + "resolved": "https://registry.npmjs.org/use-debounce/-/use-debounce-10.0.4.tgz", + "integrity": "sha512-6Cf7Yr7Wk7Kdv77nnJMf6de4HuDE4dTxKij+RqE9rufDsI6zsbjyAxcH5y2ueJCQAnfgKbzXbZHYlkFwmBlWkw==", "engines": { "node": ">= 16.0.0" }, diff --git a/frontend/package.json b/frontend/package.json index 6f83e36a..319139b0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -47,7 +47,7 @@ "rfdc": "^1.3.1", "s3-sync-client": "^4.3.1", "source-map-support": "^0.5.21", - "use-debounce": "^10.0.3", + "use-debounce": "^10.0.4", "use-immer": "^0.9.0", "uuidv7": "^0.6.3" }, diff --git a/frontend/src/components/ui/blockEditor/directoryViewer/CodeEditor.jsx b/frontend/src/components/ui/blockEditor/directoryViewer/CodeEditor.jsx index 77727bdf..1b5950ff 100644 --- a/frontend/src/components/ui/blockEditor/directoryViewer/CodeEditor.jsx +++ b/frontend/src/components/ui/blockEditor/directoryViewer/CodeEditor.jsx @@ -1,14 +1,19 @@ import { useContext } from "react"; import PromptViewer from "./PromptViewer"; import CodeManualEditor from "./CodeManualEditor"; -import { SelectedPromptContext } from "./DirectoryViewer"; +import { SelectedPromptContext, FileHandleContext } from "./DirectoryViewer"; export default function CodeEditor() { const selectedPrompt = useContext(SelectedPromptContext); + const fileHandle = useContext(FileHandleContext); return (
- {selectedPrompt.selected ? : } + {selectedPrompt.selected ? ( + + ) : ( + + )}
); } diff --git a/frontend/src/components/ui/blockEditor/directoryViewer/CodeManualEditor.jsx b/frontend/src/components/ui/blockEditor/directoryViewer/CodeManualEditor.jsx index 51a3f88e..a699a791 100644 --- a/frontend/src/components/ui/blockEditor/directoryViewer/CodeManualEditor.jsx +++ b/frontend/src/components/ui/blockEditor/directoryViewer/CodeManualEditor.jsx @@ -1,6 +1,6 @@ import { Button } from "@carbon/react"; import { EditorCodeMirror } from "./CodeMirrorComponents"; -import { useContext, useState } from "react"; +import { useContext, useEffect, useRef, useState } from "react"; import { FileBufferContext } from "./DirectoryViewer"; import { ChatHistoryContext } from "./DirectoryViewer"; import { FileHandleContext } from "./DirectoryViewer"; @@ -17,6 +17,8 @@ export default function CodeManualEditor() { const fileHandle = useContext(FileHandleContext); const fileBuffer = useContext(FileBufferContext); const chatHistory = useContext(ChatHistoryContext); + const initialCode = useRef(fileBuffer.content); + const editorRef = useRef(); const compile = useCompileComputation(); const [isLoading, setIsLoading] = useState(false); @@ -32,6 +34,28 @@ export default function CodeManualEditor() { ]); const saveDisabled = isLoading || !fileBuffer.hasPendingChanges; + useEffect(() => { + updateEditorState(); + }, [fileBuffer.content]); + + const updateEditorState = () => { + if (editorRef?.current?.view) { + const editorCode = editorRef.current.view.state.doc?.toString(); + if (editorCode !== fileBuffer.content) { + const length = editorCode.length ?? 0; + const newLength = fileBuffer.content.length; + const cursorPosition = editorRef.current.view.state.selection.main.head; + const newCursorPosition = clamp(cursorPosition, 0, newLength); + + const transaction = editorRef.current.view.state.update({ + changes: { from: 0, to: length, insert: fileBuffer.content }, + selection: { anchor: newCursorPosition }, + }); + editorRef.current.view.dispatch(transaction); + } + } + }; + const handleChange = (value) => { fileBuffer.update(value); }; @@ -49,7 +73,8 @@ export default function CodeManualEditor() { return (
@@ -69,3 +94,7 @@ export default function CodeManualEditor() {
); } + +function clamp(value, min, max) { + return Math.min(Math.max(value, min), max); +} diff --git a/frontend/src/components/ui/blockEditor/directoryViewer/CodeMirrorComponents.jsx b/frontend/src/components/ui/blockEditor/directoryViewer/CodeMirrorComponents.jsx index 1feb2cb5..eff6d7d5 100644 --- a/frontend/src/components/ui/blockEditor/directoryViewer/CodeMirrorComponents.jsx +++ b/frontend/src/components/ui/blockEditor/directoryViewer/CodeMirrorComponents.jsx @@ -3,6 +3,7 @@ import { EditorView } from "@codemirror/view"; import { loadLanguage } from "@uiw/codemirror-extensions-langs"; import { githubLight } from "@uiw/codemirror-theme-github"; import { vscodeDark } from "@uiw/codemirror-theme-vscode"; +import { forwardRef } from "react"; import CodeMirror from "@uiw/react-codemirror"; import { atom, useAtom } from "jotai"; @@ -70,33 +71,36 @@ export const ViewerCodeMirror = ({ currentIndex, code }) => { ); }; -export const EditorCodeMirror = ({ code, onChange, keymap }) => { - const [theme] = useAtom(themeAtom); - const extensions = [loadLanguage("python"), backgroundTheme, keymap]; +export const EditorCodeMirror = forwardRef( + ({ code, onChange, keymap }, ref) => { + const [theme] = useAtom(themeAtom); + const extensions = [loadLanguage("python"), backgroundTheme, keymap]; - return ( - - ); -}; + return ( + + ); + }, +); export const LogsCodeMirror = ({ code, onUpdate }) => { const [theme] = useAtom(themeAtom); diff --git a/frontend/src/hooks/useConfirmModal.jsx b/frontend/src/hooks/useConfirmModal.jsx index 7132f9b6..ffe195c4 100644 --- a/frontend/src/hooks/useConfirmModal.jsx +++ b/frontend/src/hooks/useConfirmModal.jsx @@ -15,7 +15,7 @@ export default function useConfirmModal() { return; } - fileHandle.set(selectedFile); + await fileHandle.set(selectedFile); }; const close = () => { @@ -24,12 +24,12 @@ export default function useConfirmModal() { const save = async () => { fileBuffer.save(); - fileHandle.set(selectedFile); + await fileHandle.set(selectedFile); close(); }; const discard = async () => { - fileHandle.set(selectedFile); + await fileHandle.set(selectedFile); close(); }; diff --git a/frontend/src/hooks/useFileBuffer.jsx b/frontend/src/hooks/useFileBuffer.jsx index a8fc0a6c..ef7a2f1e 100644 --- a/frontend/src/hooks/useFileBuffer.jsx +++ b/frontend/src/hooks/useFileBuffer.jsx @@ -1,14 +1,16 @@ import { trpc } from "@/utils/trpc"; -import { useState, useRef } from "react"; +import { useState } from "react"; +import { useDebouncedCallback } from "use-debounce"; export default function useFileBuffer(pipelinePath, blockId, relativePath) { const getFileContent = trpc.block.file.byPath.get.useMutation(); const updateFileContent = trpc.block.file.byPath.update.useMutation(); - const fileContentBuffer = useRef(""); + const [content, setContent] = useState(""); + const setContentDebounced = useDebouncedCallback(setContent, 100); const [hasPendingChanges, setHasPendingChanges] = useState(false); const update = (newValue) => { - fileContentBuffer.current = newValue; + setContentDebounced(newValue); setHasPendingChanges(true); }; @@ -17,13 +19,13 @@ export default function useFileBuffer(pipelinePath, blockId, relativePath) { pipelinePath, blockId: blockId, relPath: relativePath, - content: fileContentBuffer.current, + content: content, }); setHasPendingChanges(false); }; const updateSave = async (newValue) => { - fileContentBuffer.current = newValue; + setContent(newValue); await updateFileContent.mutateAsync({ pipelinePath, blockId: blockId, @@ -34,18 +36,17 @@ export default function useFileBuffer(pipelinePath, blockId, relativePath) { }; const load = async (relativePath) => { - fileContentBuffer.current = ""; const content = await getFileContent.mutateAsync({ pipelinePath: pipelinePath, blockId: blockId, relPath: relativePath, }); setHasPendingChanges(false); - fileContentBuffer.current = content; + setContent(content); }; return { - content: fileContentBuffer.current, + content, hasPendingChanges, update, load, diff --git a/frontend/src/hooks/useFileHandle.jsx b/frontend/src/hooks/useFileHandle.jsx index 353f1e11..6f926824 100644 --- a/frontend/src/hooks/useFileHandle.jsx +++ b/frontend/src/hooks/useFileHandle.jsx @@ -12,10 +12,10 @@ export default function useFileHandle(pipelinePath, blockId) { ); const set = async (file) => { - setCurrentFile(file); if (file.read) { await buffer.load(file.relativePath); } + setCurrentFile(file); }; return { From cf3ecbaaa965654b0b3e3b8e5fbe9d0d225d3a51 Mon Sep 17 00:00:00 2001 From: Maharaj Teertha Deb <86840917+TeerthaDeb@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:31:47 -0400 Subject: [PATCH 4/4] File block (#235) * "Added FolderBlock component and updated conditional statement to render it for 'folder' type" * "Added FolderBlock component and updated conditional statement to render it for 'folder' type" * Added MultiFileBlock component * Added MultiFileBlock component * updates on multi-file block * Update folder upload block specs and BlockGenerator component * Updated email and github of Zetane * try 1 * try 1 * folder and multi-file works, with minor bugs * folder upload block shows it's folders name. * folder-upload and multi-files upload working fine. * got rid of localStorage * Refactor file path handling in uploadBlocks function to use relative paths and preserve directory structure. * Refactor MultiFileBlock component and remove unused functions and imports from pipelineSerialization.js and MultiFileBlock.jsx files. * Refactor pipeline serialization and MultiFileBlock component --------- Co-authored-by: Jon Magoon --- frontend/server/pipelineSerialization.js | 18 ++++++++++++++---- .../ui/blockGenerator/MultiFileBlock.jsx | 11 +---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/frontend/server/pipelineSerialization.js b/frontend/server/pipelineSerialization.js index 7da03550..ab5683cd 100644 --- a/frontend/server/pipelineSerialization.js +++ b/frontend/server/pipelineSerialization.js @@ -239,16 +239,26 @@ async function uploadBlocks( const cleanedValue = param.value.replace(/\\/g, "\\\\"); const fileNames = []; const filePaths = JSON.parse(cleanedValue); + const firstFilePath = filePaths[0]; + const pathSegments = firstFilePath.split(/[/\\]/); + const rootFolder = pathSegments[pathSegments.length - 2]; for (const filePath of filePaths) { // console.log("Uploading file:", filePath); // Debugging log - const fileName = path.basename(filePath); - fileNames.push(fileName); - const awsKey = `${pipelineId}/${executionId}/${fileName}`; + let relativePath = filePath.split(rootFolder)[1]; + + if (param.type === "folder") { + relativePath = relativePath.replace(/\\/g, "/").trim(); + } else { + relativePath = relativePath.replace(/\\/g, "").trim(); + } + + fileNames.push(relativePath); + const awsKey = `${pipelineId}/${executionId}/${relativePath}`; if (filePath && filePath.trim()) { await checkAndUpload(awsKey, filePath, anvilConfiguration); - // console.log(`Uploaded: ${fileName} to ${awsKey}`); // which file uploaded. + // console.log(`Uploaded: ${relativePath} to ${awsKey}`); // which file uploaded. } else { // log invalid paths. console.error("Invalid file path:", filePath); diff --git a/frontend/src/components/ui/blockGenerator/MultiFileBlock.jsx b/frontend/src/components/ui/blockGenerator/MultiFileBlock.jsx index dabdf93e..4f794d6c 100644 --- a/frontend/src/components/ui/blockGenerator/MultiFileBlock.jsx +++ b/frontend/src/components/ui/blockGenerator/MultiFileBlock.jsx @@ -41,11 +41,8 @@ export const MultiFileBlock = ({ blockId, block, setFocusAction, history }) => { const loadFiles = () => { const files = Array.from(fileInput.current.files); - // console.log("FILES: " ,files) const filePaths = processFiles(files); - // console.log("filePaths:" , filePaths) const formattedValue = `[${filePaths.map((file) => `"${file}"`).join(", ")}]`; - // console.log("formatted value: " , formattedValue) setFocusAction((draft) => { draft.data[blockId].action.parameters["files"].value = formattedValue; // Update to formatted value draft.data[blockId].action.parameters["files"].type = "file[]"; @@ -58,12 +55,7 @@ export const MultiFileBlock = ({ blockId, block, setFocusAction, history }) => {
{renderedFiles.map((file, index) => ( -
- {file} -
+
{file}
))}
{ multiple parameters-path="true" style={{ - // for better visibility. color: "#ffffff", }} />