fixed file downloads (#261)

* fixed file downloads

* correct spec location

* remove logs

* ignore history folder

* remove console.log

frontend/server/execution.js

@@ -9,11 +9,13 @@ export async function syncExecutionResults(
   executionUuid,
   anvilConfiguration,
   merkle,
+  spec,
 ) {
   let s3Prefix;
+  let org;
   if (anvilConfiguration.anvil.token) {
     const data = atob(anvilConfiguration.anvil.token.split(".")[1]);
-    const org = JSON.parse(data).sub;
+    org = JSON.parse(data).sub;
     s3Prefix = `${org}/${pipelineUuid}/${executionUuid}`;
   } else {
     s3Prefix = `${pipelineUuid}/${executionUuid}`;
@@ -28,10 +30,13 @@ export async function syncExecutionResults(
       try {
         // for downloading files
         for (const blockKey in merkle_persed.blocks) {
+          const blockSpec = spec[blockKey];
+          if (!org || !blockSpec?.action?.container?.image) {
+            continue;
+          }
           const block = merkle_persed.blocks[blockKey];
           const blockPath = localPath.split("history")[0];
-          const blockName = blockKey.split("-").slice(0, -1).join("-");
-          const blocksS3Prefix = `${blockName}-${block.hash}-build`;
+          const blocksS3Prefix = `${org}/${blockSpec?.action?.container?.image}-${block.hash}-build`;
           const localBlockDir = path.join(blockPath, blockKey);
 
           if (!fs.existsSync(localBlockDir)) {
@@ -45,7 +50,6 @@ export async function syncExecutionResults(
               localBlockDir,
               anvilConfiguration,
             );
-            // console.log(`Downloaded folder: ${blocksS3Prefix} to ${localBlockDir}`);
           } catch (err) {
             console.error("Error downloading the folder:", err);
           }

frontend/server/git.js

@@ -3,6 +3,24 @@ import fs from "fs";
 import path from "path";
 import { cacheJoin } from "./cache.js";
 
+async function ensureGitignore(dir) {
+  const gitignorePath = path.join(dir, ".gitignore");
+  try {
+    // Check if .gitignore exists
+    await fs.promises.access(gitignorePath);
+    const content = await fs.promises.readFile(gitignorePath, "utf8");
+    if (!content.includes("history/")) {
+      await fs.promises.appendFile(gitignorePath, "\nhistory/\n");
+      return true; // indicates .gitignore was modified
+    }
+    return false; // indicates no changes were needed
+  } catch (e) {
+    // .gitignore doesn't exist, create it
+    await fs.promises.writeFile(gitignorePath, "history/\n", "utf8");
+    return true; // indicates .gitignore was created
+  }
+}
+
 export async function ensureGitRepoAndCommitBlocks(
   buildContextStatuses,
   buildPath,
@@ -14,7 +32,7 @@ export async function ensureGitRepoAndCommitBlocks(
     await Promise.all(
       buildContextStatuses
         .filter((context) => {
-          context.hash != "";
+          return context.hash != "";
         })
         .map(async ({ blockKey }) => {
           const sourcePath = path.join(buildPath, blockKey);
@@ -64,6 +82,24 @@ export async function ensureGitRepoAndCommitBlocks(
     });
   }
 
+  const gitignoreModified = await ensureGitignore(cachePath);
+
+  if (gitignoreModified) {
+    // Add and commit .gitignore if it was created or modified
+    await git.add({ fs, dir: cachePath, filepath: ".gitignore" });
+    await git.commit({
+      fs,
+      dir: cachePath,
+      message: isRepo
+        ? "chore: update .gitignore"
+        : "chore: initial commit with .gitignore",
+      author: {
+        name: "Pipeline System",
+        email: "[email protected]",
+      },
+    });
+  }
+
   // 3. Check for changes
   const statusMatrix = await git.statusMatrix({ fs, dir: cachePath });
   const hasChanges = statusMatrix.some(

frontend/server/router.ts

@@ -285,21 +285,24 @@ export const appRouter = router({
       pipelineUuid: z.string(),
       executionUuid: z.string(),
       anvilConfiguration: anvilConfigurationSchema,
-      merkle: z.string(),
+      merkle: z.string().optional(),
+      spec: z.any().optional(),
     }),
     async ({
       pipelinePath,
       pipelineUuid,
       executionUuid,
       anvilConfiguration,
       merkle,
+      spec,
     }) =>
       syncExecutionResults(
         pipelinePath,
         pipelineUuid,
         executionUuid,
         anvilConfiguration,
         merkle,
+        spec,
       ),
   ),
 

frontend/server/s3.js

@@ -93,13 +93,14 @@ export async function uploadDirectory(key, diretoryPath, anvilConfiguration) {
 
 async function upload(key, filePath, anvilConfiguration) {
   const client = getClient(anvilConfiguration);
+  const s3key = getFullS3Key(key, anvilConfiguration);
 
   try {
     const fileBody = await fs.readFile(filePath);
     const res = await client.send(
       new PutObjectCommand({
         Bucket: anvilConfiguration.s3.bucket,
-        Key: getFullS3Key(key, anvilConfiguration),
+        Key: s3key,
         Body: fileBody,
       }),
     );

frontend/src/components/ui/WorkspaceFetcher.jsx

@@ -58,7 +58,7 @@ export default function WorkspaceFetcher() {
     setPipelines(updatedPipelines);
     if (isActive) {
       try {
-        const merkle = "undefined"; // while running Merkle should be a blank something.
+        const merkle = undefined; // while running Merkle should be a blank something.
         await syncResults(key, merkle);
         // TODO:
         // Surface sync errors

frontend/src/hooks/useExecutionResults.jsx

@@ -16,6 +16,7 @@ export const useSyncExecutionResults = () => {
       executionUuid: pipeline.record.Execution,
       anvilConfiguration: configuration,
       merkle: merkle,
+      spec: pipeline.data,
     });
   };