Skip to content

Commit

Permalink
Add Phase 2 COSI Support (#85)
Browse files Browse the repository at this point in the history 
<!-- Description: Please provide a summary of the changes and the
motivation behind them. -->

PR summary:

This is a phase 2 COSI support, we should now be able to generate cosi
file from a **_non-verity_** base image with a verity config, and the
verity information will be included in the cosi metadata file, so it
will align with expected sample format

local test:
tested with command `sudo ./imagecustomizer --log-level debug
--build-dir ./build --image-file core-3.0.20241220.vhdx
--output-image-file ./output-image-test.cosi --output-image-format cosi
--config-file ../pkg/imagecustomizerlib/testdata/verity-config.yaml`
unpacked the .cosi file

![image](https://github.com/user-attachments/assets/9c69f926-56af-4167-bbe0-1dbcc3374827)
and obtained output/metadata.json

```
{
  "version": "1.0",
  "osArch": "amd64",
  "images": [
    {
      "image": {
        "path": "images/output-image-test_1.raw.zst",
        "compressedSize": 988946,
        "uncompressedSize": 8388608,
        "sha384": "6c2a78a9bc84624d90fdf2b0be7e33dd4abd305f8ae2467a4868bdf3ed8114cbe72c8512e5836e0a806e932732f79585"
      },
      "mountPoint": "",
      "fsType": "vfat",
      "fsUuid": "1163-42B5",
      "partType": "c12a7328-f81f-11d2-ba4b-00a0c93ec93b",
      "verity": null
    },
    {
      "image": {
        "path": "images/output-image-test_2.raw.zst",
        "compressedSize": 51437979,
        "uncompressedSize": 1064304640,
        "sha384": "f6594fa455d3feffd002dddd32afc843856e09bb850c6a3a6ec84366a00c8dde92e28f68a464ca62e8160bb5ad6a50e2"
      },
      "mountPoint": "",
      "fsType": "ext4",
      "fsUuid": "14e039a8-61d2-4d43-85b2-f26c92326c03",
      "partType": "0fc63daf-8483-4772-8e79-3d69d8477de4",
      "verity": null
    },
    {
      "image": {
        "path": "images/output-image-test_3.raw.zst",
        "compressedSize": 115309290,
        "uncompressedSize": 2147483648,
        "sha384": "29d8663dbd8f320ad9bba8169173837213eb973842f52ec774af8840f130c596b118cef88c2a4c38bc8820cfba482637"
      },
      "mountPoint": "",
      "fsType": "ext4",
      "fsUuid": "393703a2-b8a9-4574-8fd0-d961674c3efa",
      "partType": "0fc63daf-8483-4772-8e79-3d69d8477de4",
      "verity": {
        "image": {
          "path": "images/output-image-test_4.raw.zst",
          "compressedSize": 2908417,
          "uncompressedSize": 134217728,
          "sha384": "d4c8dd1357868b9aab271af8f74c52f05c8e4378d53637644cec43fc7a20adbb0aa1f0b01d701555cfda0490205d11ab"
        },
        "hash": "d5ff43f8d03a738b1e25f4eeb1b1f71441fa456daf60a664f64a2e95316ec65f"
      }
    },
    {
      "image": {
        "path": "images/output-image-test_5.raw.zst",
        "compressedSize": 28775661,
        "uncompressedSize": 2012217344,
        "sha384": "9a69d82c2e1c0b6681191cbcc6847ba8db2b54f15671b0b9a00e1649ce753038a8f2cb2b6d7e1bc36bdcc9cc8d0dbbf1"
      },
      "mountPoint": "",
      "fsType": "ext4",
      "fsUuid": "8fee3677-b8ba-432b-8add-3f336d11575f",
      "partType": "0fc63daf-8483-4772-8e79-3d69d8477de4",
      "verity": null
    }
  ],
  "osRelease": "",
  "id": "85e18600-1160-1b53-bcea-d93a3f0d848b"
```

Note: 
A verity base image currently is not supported, will wait customizations
on verity images to be enabled. Also, will have a follow-up pr to fix
the missing mountpoints.

---

### **Checklist**
- [ ] Tests added/updated
- [ ] Documentation updated (if needed)
- [ ] Code conforms to style guidelines
  • Loading branch information
@elainezhao96
elainezhao96 authored Jan 23, 2025
1 parent 5d3b963 commit 1188074
Show file tree
Hide file tree
Showing 3 changed files with 197 additions and 80 deletions.
191 changes: 141 additions & 50 deletions toolkit/tools/pkg/imagecustomizerlib/cosicommon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package imagecustomizerlib
import (
"archive/tar"
"crypto/sha512"
_ "embed"
"encoding/json"
"fmt"
"io"
Expand All @@ -17,9 +16,10 @@ import (
)

type ImageBuildData struct {
Source string
KnownInfo outputPartitionMetadata
Metadata *Image
Source string
KnownInfo outputPartitionMetadata
Metadata *Image
VeritySource string
}

func convertToCosi(ic *ImageCustomizerParameters) error {
Expand All @@ -41,9 +41,9 @@ func convertToCosi(ic *ImageCustomizerParameters) error {
return err
}

err = buildCosiFile(outputDir, ic.outputImageFile, partitionMetadataOutput, ic.imageUuidStr)
err = buildCosiFile(outputDir, ic.outputImageFile, partitionMetadataOutput, ic.verityMetadata, ic.imageUuidStr)
if err != nil {
return fmt.Errorf("failed to build COSI:\n%w", err)
return fmt.Errorf("failed to build COSI file:\n%w", err)
}

logger.Log.Infof("Successfully converted to COSI: %s", ic.outputImageFile)
Expand All @@ -56,45 +56,90 @@ func convertToCosi(ic *ImageCustomizerParameters) error {
return nil
}

func buildCosiFile(sourceDir string, outputFile string, expectedImages []outputPartitionMetadata, imageUuidStr string) error {
metadata := MetadataJson{
Version: "1.0",
OsArch: runtime.GOARCH,
Id: imageUuidStr,
Images: make([]Image, len(expectedImages)),
func buildCosiFile(sourceDir string, outputFile string, partitions []outputPartitionMetadata, verityMetadata []verityDeviceMetadata, imageUuidStr string) error {

// Pre-compute a map for quick lookup of partition metadata by UUID
partUuidToMetadata := make(map[string]outputPartitionMetadata)
for _, partition := range partitions {
partUuidToMetadata[partition.PartUuid] = partition
}

if len(expectedImages) == 0 {
return fmt.Errorf("no images to build")
// Pre-compute a set of verity hash UUIDs for quick lookup
verityHashUuids := make(map[string]struct{})
for _, verity := range verityMetadata {
verityHashUuids[verity.hashPartUuid] = struct{}{}
}

// Create an interim metadata struct to combine the known data with the metadata
imageData := make([]ImageBuildData, len(expectedImages))
for i, image := range expectedImages {
metadata := &metadata.Images[i]
imageData[i] = ImageBuildData{
Source: path.Join(sourceDir, image.PartitionFilename),
Metadata: metadata,
KnownInfo: image,
imageData := []ImageBuildData{}

for _, partition := range partitions {
// Skip verity hash partitions as their metadata will be assigned to the corresponding data partitions
if _, isVerityHash := verityHashUuids[partition.PartUuid]; isVerityHash {
continue
}

metadataImage := Image{
Image: ImageFile{
Path: path.Join("images", partition.PartitionFilename),
UncompressedSize: partition.UncompressedSize,
},
PartType: partition.PartitionTypeUuid,
MountPoint: partition.Mountpoint,
FsType: partition.FileSystemType,
FsUuid: partition.Uuid,
}

imageDataEntry := ImageBuildData{
Source: path.Join(sourceDir, partition.PartitionFilename),
Metadata: &metadataImage,
KnownInfo: partition,
}

// Add Verity metadata if the partition has a matching entry in verityMetadata
for _, verity := range verityMetadata {
if partition.PartUuid == verity.dataPartUuid {
hashPartition, exists := partUuidToMetadata[verity.hashPartUuid]
if !exists {
return fmt.Errorf("missing metadata for hash partition UUID:\n%s", verity.hashPartUuid)
}

metadataImage.Verity = &Verity{
Hash: verity.hash,
Image: ImageFile{
Path: path.Join("images", hashPartition.PartitionFilename),
UncompressedSize: hashPartition.UncompressedSize,
},
}

veritySourcePath := path.Join(sourceDir, hashPartition.PartitionFilename)
imageDataEntry.VeritySource = veritySourcePath
break
}
}

metadata.Image.Path = path.Join("images", image.PartitionFilename)
metadata.PartType = image.PartitionTypeUuid
metadata.MountPoint = image.Mountpoint
metadata.FsType = image.FileSystemType
metadata.FsUuid = image.Uuid
metadata.UncompressedSize = image.UncompressedSize
imageData = append(imageData, imageDataEntry)
}

// Populate metadata for each image
for _, data := range imageData {
logger.Log.Infof("Processing image %s", data.Source)
err := populateMetadata(data)
for i := range imageData {
err := populateMetadata(&imageData[i])
if err != nil {
return fmt.Errorf("failed to populate metadata for %s:\n%w", data.Source, err)
return fmt.Errorf("failed to populate metadata for %s:\n%w", imageData[i].Source, err)
}

logger.Log.Infof("Populated metadata for image %s", data.Source)
logger.Log.Infof("Populated metadata for image %s", imageData[i].Source)
}

metadata := MetadataJson{
Version: "1.0",
OsArch: runtime.GOARCH,
Id: imageUuidStr,
Images: make([]Image, len(imageData)),
}

// Copy updated metadata
for i, data := range imageData {
metadata.Images[i] = *data.Metadata
}

// Marshal metadata.json
Expand Down Expand Up @@ -133,26 +178,43 @@ func buildCosiFile(sourceDir string, outputFile string, expectedImages []outputP
}

func addToCosi(data ImageBuildData, tw *tar.Writer) error {
imageFile, err := os.Open(data.Source)
err := addFileToCosi(tw, data.Source, data.Metadata.Image)
if err != nil {
return fmt.Errorf("failed to add image file to COSI:\n%w", err)
}

if data.VeritySource != "" && data.Metadata.Verity != nil {
err := addFileToCosi(tw, data.VeritySource, data.Metadata.Verity.Image)
if err != nil {
return fmt.Errorf("failed to add verity file to COSI:\n%w", err)
}
}

return nil
}

func addFileToCosi(tw *tar.Writer, source string, image ImageFile) error {
file, err := os.Open(source)
if err != nil {
return fmt.Errorf("failed to open image file:\n%w", err)
return fmt.Errorf("failed to open file :\n%w", err)
}
defer imageFile.Close()
defer file.Close()

err = tw.WriteHeader(&tar.Header{
Typeflag: tar.TypeReg,
Name: data.Metadata.Image.Path,
Size: int64(data.Metadata.Image.CompressedSize),
Name: image.Path,
Size: int64(image.CompressedSize),
Mode: 0o400,
Format: tar.FormatPAX,
})

if err != nil {
return fmt.Errorf("failed to write tar header:\n%w", err)
return fmt.Errorf("failed to write tar header for file '%s':\n%w", image.Path, err)
}

_, err = io.Copy(tw, imageFile)
_, err = io.Copy(tw, file)
if err != nil {
return fmt.Errorf("failed to write image to COSI:\n%w", err)
return fmt.Errorf("failed to write image '%s' to COSI:\n%w", image.Path, err)
}

return nil
Expand All @@ -172,21 +234,50 @@ func sha384sum(path string) (string, error) {
return fmt.Sprintf("%x", sha384.Sum(nil)), nil
}

func populateMetadata(data ImageBuildData) error {
stat, err := os.Stat(data.Source)
func populateImageFile(source string, imageFile *ImageFile) error {
stat, err := os.Stat(source)
if err != nil {
return fmt.Errorf("filed to stat %s:\n%w", data.Source, err)
return fmt.Errorf("failed to stat %s:\n%w", source, err)
}
if stat.IsDir() {
return fmt.Errorf("%s is a directory", data.Source)
return fmt.Errorf("%s is a directory", source)
}
data.Metadata.Image.CompressedSize = uint64(stat.Size())
imageFile.CompressedSize = uint64(stat.Size())

// Calculate the sha384 of the image
sha384, err := sha384sum(data.Source)
sha384, err := sha384sum(source)
if err != nil {
return fmt.Errorf("failed to calculate sha384 of %s:\n%w", data.Source, err)
return fmt.Errorf("failed to calculate sha384 of %s:\n%w", source, err)
}
imageFile.Sha384 = sha384

return nil
}

// Enriches the image metadata with size and checksum
func populateMetadata(data *ImageBuildData) error {
if err := populateImageFile(data.Source, &data.Metadata.Image); err != nil {
return fmt.Errorf("failed to populate metadata:\n%w", err)
}

if err := populateVerityMetadata(data.VeritySource, data.Metadata.Verity); err != nil {
return fmt.Errorf("failed to populate verity metadata:\n%w", err)
}
data.Metadata.Image.Sha384 = sha384

return nil
}

func populateVerityMetadata(source string, verity *Verity) error {
if source == "" && verity == nil {
return nil
}

if source == "" || verity == nil {
return fmt.Errorf("verity source and verity metadata must be both defined or both undefined")
}

if err := populateImageFile(source, &verity.Image); err != nil {
return fmt.Errorf("failed to populate verity image metadata:\n%w", err)
}

return nil
}
17 changes: 8 additions & 9 deletions toolkit/tools/pkg/imagecustomizerlib/cosimetadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,17 @@ type MetadataJson struct {
}

type Image struct {
Image ImageFile `json:"image"`
MountPoint string `json:"mountPoint"`
FsType string `json:"fsType"`
FsUuid string `json:"fsUuid"`
PartType string `json:"partType"`
Verity *Verity `json:"verity"`
UncompressedSize uint64 `json:"uncompressedSize"`
Image ImageFile `json:"image"`
MountPoint string `json:"mountPoint"`
FsType string `json:"fsType"`
FsUuid string `json:"fsUuid"`
PartType string `json:"partType"`
Verity *Verity `json:"verity"`
}

type Verity struct {
Image ImageFile `json:"image"`
Roothash string `json:"roothash"`
Image ImageFile `json:"image"`
Hash string `json:"hash"`
}

type ImageFile struct {
Expand Down
Loading

0 comments on commit 1188074

Please sign in to comment.