Skip to content

Commit

Permalink
fix(toolkit): use random string when uploading to gcp/r2 (#257)
Browse files Browse the repository at this point in the history
  • Loading branch information
efiop authored Jul 13, 2024
1 parent 5677f46 commit 4b493fd
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 41 deletions.
7 changes: 6 additions & 1 deletion projects/fal/src/fal/toolkit/file/providers/gcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import datetime
import json
import os
import posixpath
import uuid
from dataclasses import dataclass

from fal.toolkit.file.types import FileData, FileRepository
Expand Down Expand Up @@ -49,7 +51,10 @@ def bucket(self):
return self._bucket

def save(self, data: FileData) -> str:
destination_path = os.path.join(self.folder, data.file_name)
destination_path = posixpath.join(
self.folder,
f"{uuid.uuid4().hex}_{data.file_name}",
)

gcp_blob = self.bucket.blob(destination_path)
gcp_blob.upload_from_string(data.data, content_type=data.content_type)
Expand Down
7 changes: 6 additions & 1 deletion projects/fal/src/fal/toolkit/file/providers/r2.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import json
import os
import posixpath
import uuid
from dataclasses import dataclass
from io import BytesIO

Expand Down Expand Up @@ -66,7 +68,10 @@ def bucket(self):
return self._bucket

def save(self, data: FileData) -> str:
destination_path = os.path.join(self.key, data.file_name)
destination_path = posixpath.join(
self.key,
f"{uuid.uuid4().hex}_{data.file_name}",
)

s3_object = self.bucket.Object(destination_path)
s3_object.upload_fileobj(
Expand Down
40 changes: 1 addition & 39 deletions projects/fal/tests/toolkit/file_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@
from base64 import b64encode

import pytest
from fal.toolkit.file.file import BUILT_IN_REPOSITORIES, File, GoogleStorageRepository
from fal.toolkit.file.types import FileRepository, RepositoryId
from fal.toolkit.file.file import File, GoogleStorageRepository


def test_binary_content_matches():
Expand Down Expand Up @@ -65,40 +64,3 @@ def test_gcp_storage_if_available():
assert file.url.startswith(
"https://storage.googleapis.com/fal_registry_image_results/"
)


@pytest.mark.xfail(reason="fal cdn is temporarily broken")
@pytest.mark.parametrize(
"repo",
BUILT_IN_REPOSITORIES.keys(),
)
def test_uniqueness_of_file_name(repo: RepositoryId | FileRepository):
if repo == "in_memory":
return

if repo == "gcp_storage":
gcp_sa_json = os.environ.get("GCLOUD_SA_JSON")
if gcp_sa_json is None:
pytest.skip(reason="GCLOUD_SA_JSON environment variable is not set")
repo = GoogleStorageRepository(bucket_name="fal_registry_image_results")

if repo == "r2":
r2_account_json = os.environ.get("R2_CREDS_JSON")
if r2_account_json is None:
pytest.skip(reason="R2_CREDS_JSON environment variable is not set")

if repo == "fal":
fal_key = os.environ.get("FAL_KEY")
if fal_key is None:
pytest.skip(reason="FAL_KEY environment variable is not set")

file = File.from_bytes(b"print('Hello!')", repository=repo, file_name="hello.py")

host_and_path = file.url.split("?")[0]
last_path = host_and_path.split("/")[-1]
assert last_path.endswith(
".py"
), f"The file name {last_path} should end with the same extension"
assert (
len(last_path) > 10
), f"There should be a long enough random string in the file name {last_path}"

0 comments on commit 4b493fd

Please sign in to comment.