Skip to content

Commit

Permalink
url: add azure blob fetching support for ignition files
Browse files Browse the repository at this point in the history
use azure sdk to authorize, initiate and fetch ignition config file from azure blob storage.

fixes: https://issues.redhat.com/browse/COS-2859
  • Loading branch information
prestist committed Dec 6, 2024
1 parent 3febddd commit c4391ea
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 1 deletion.
2 changes: 2 additions & 0 deletions docs/release-notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ Starting with this release, ignition-validate binaries are signed with the

### Features

- Add Azure blob support for fetching ignition configs

### Changes

### Bug fixes
Expand Down
66 changes: 65 additions & 1 deletion internal/resource/url.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ import (
"golang.org/x/oauth2/google"
"google.golang.org/api/option"

"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/arn"
"github.com/aws/aws-sdk-go/aws/awserr"
Expand Down Expand Up @@ -149,7 +151,11 @@ func (f *Fetcher) FetchToBuffer(u url.URL, opts FetchOptions) ([]byte, error) {
dest := new(bytes.Buffer)
switch u.Scheme {
case "http", "https":
err = f.fetchFromHTTP(u, dest, opts)
if strings.HasSuffix(u.Host, ".blob.core.windows.net") {
err = f.fetchFromAzureBlob(u, dest, opts)
} else {
err = f.fetchFromHTTP(u, dest, opts)
}
case "tftp":
err = f.fetchFromTFTP(u, dest, opts)
case "data":
Expand Down Expand Up @@ -210,6 +216,9 @@ func (f *Fetcher) Fetch(u url.URL, dest *os.File, opts FetchOptions) error {

switch u.Scheme {
case "http", "https":
if strings.HasSuffix(u.Host, ".blob.core.windows.net") {
return f.fetchFromAzureBlob(u, dest, opts)
}
return f.fetchFromHTTP(u, dest, opts)
case "tftp":
return f.fetchFromTFTP(u, dest, opts)
Expand Down Expand Up @@ -554,6 +563,61 @@ func (f *Fetcher) fetchFromS3WithCreds(ctx context.Context, dest s3target, input
return nil
}

func (f *Fetcher) fetchFromAzureBlob(u url.URL, dest io.Writer, opts FetchOptions) error {
// Read about NewDefaultAzureCredential https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#DefaultAzureCredential
// DefaultAzureCredential is a default credential chain for applications deployed to azure.
cred, err := azidentity.NewDefaultAzureCredential(nil)
if err != nil {
f.Logger.Debug("failed to obtain Azure credential: %v", err)
return fmt.Errorf("failed to obtain Azure credential: %w", err)
}

// Create a context with timeout
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()

// Breakdown the URL into storage account, container, and file
storageAccount := fmt.Sprintf("%s://%s/", u.Scheme, u.Host)
pathSegments := strings.Split(strings.Trim(u.Path, "/"), "/")
if len(pathSegments) < 2 {
f.Logger.Debug("invalid URL path: %s", u.Path)
return fmt.Errorf("invalid URL path: %s", u.Path)
}
container := pathSegments[0]
file := pathSegments[1]

// Create Azure Blob Storage client
storageClient, err := azblob.NewClient(storageAccount, cred, nil)
if err != nil {
f.Logger.Debug("failed to create azblob client: %v", err)
return fmt.Errorf("failed to create azblob client: %w", err)
}

// Download the blob with retry logic
var downloadStream azblob.DownloadStreamResponse
for i := 0; i < 3; i++ {
downloadStream, err = storageClient.DownloadStream(ctx, container, file, nil)
if err == nil {
break
}
f.Logger.Debug("error downloading blob (attempt %d): %v", i+1, err)
time.Sleep(time.Duration(i+1) * time.Second)
}
if err != nil {
return fmt.Errorf("failed to download blob from container '%s', file '%s': %w", container, file, err)
}
defer downloadStream.Body.Close()

// Process the downloaded blob
err = f.decompressCopyHashAndVerify(dest, downloadStream.Body, opts)
if err != nil {
f.Logger.Debug("Error processing downloaded blob: %v", err)
return fmt.Errorf("failed to process downloaded blob: %w", err)
}

return nil
}

// uncompress will wrap the given io.Reader in a decompresser specified in the
// FetchOptions, and return an io.ReadCloser with the decompressed data stream.
func (f *Fetcher) uncompress(r io.Reader, opts FetchOptions) (io.ReadCloser, error) {
Expand Down

0 comments on commit c4391ea

Please sign in to comment.