Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ require (
github.com/containerd/typeurl/v2 v2.2.3
github.com/google/go-cmp v0.7.0
github.com/google/go-containerregistry v0.21.5
github.com/google/uuid v1.6.0
github.com/linuxkit/virtsock v0.0.0-20241009230534-cb6a20cc0422
github.com/mattn/go-shellwords v1.0.12
github.com/moby/sys/user v0.4.0
Expand Down Expand Up @@ -99,6 +98,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/mux v1.8.1 // indirect
github.com/josephspurrier/goversioninfo v1.5.0 // indirect
github.com/klauspost/compress v1.18.5 // indirect
Expand Down
2 changes: 1 addition & 1 deletion internal/builder/vm/lcow/confidential.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func parseConfidentialOptions(
}
log.G(ctx).WithField("dmVerityRootfsPath", dmVerityRootfsTemplatePath).Debug("DM Verity rootfs path configured")

// Note: VPMem and vPCI assigned devices are already disabled in parseDeviceOptions
// Note: vPCI assigned devices are already disabled in parseDeviceOptions
// when isConfidential is true.

chipset := &hcsschema.Chipset{}
Expand Down
71 changes: 12 additions & 59 deletions internal/builder/vm/lcow/devices.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"fmt"
"path/filepath"
"runtime"
"strings"

"github.com/Microsoft/hcsshim/internal/controller/device/vpci"
hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2"
Expand All @@ -24,7 +23,7 @@ import (
)

// parseDeviceOptions parses device options from annotations and assigned devices.
// isConfidential indicates if this is a confidential scenario, which affects VPMem and PCI device configuration.
// isConfidential indicates if this is a confidential scenario, which affects PCI device configuration.
// numaConfig is used to determine if NUMA affinity propagation should be enabled for vPCI devices.
func parseDeviceOptions(
ctx context.Context,
Expand All @@ -34,7 +33,7 @@ func parseDeviceOptions(
isNumaEnabled bool,
isFullyPhysicallyBacked bool,
isConfidential bool,
) (*hcsschema.VirtualPMemController, map[string]hcsschema.Scsi, map[string]hcsschema.VirtualPciDevice, error) {
) (map[string]hcsschema.Scsi, map[string]hcsschema.VirtualPciDevice, error) {

log.G(ctx).WithFields(logrus.Fields{
"deviceCount": len(devices),
Expand All @@ -43,64 +42,16 @@ func parseDeviceOptions(
}).Debug("parseDeviceOptions: starting device options parsing")

// ===============================Parse VPMem configuration===============================
vpmemCount := oci.ParseAnnotationsUint32(ctx, annotations, shimannotations.VPMemCount, vmutils.DefaultVPMEMCount)
vpmemSize := oci.ParseAnnotationsUint64(ctx, annotations, shimannotations.VPMemSize, vmutils.DefaultVPMemSizeBytes)
// By default, we should set vpmem count to 0.
vpmemCount := oci.ParseAnnotationsUint32(ctx, annotations, shimannotations.VPMemCount, 0)

// VPMem is not supported by the enlightened kernel for SNP (confidential VMs, and Hyper-V on arm64).
// Todo: Remove arm64 check once VPMem is supported by Hyper-V on arm64.
if isFullyPhysicallyBacked || isConfidential || runtime.GOARCH == "arm64" {
vpmemCount = 0
}

if vpmemCount > vmutils.MaxVPMEMCount {
return nil, nil, nil, fmt.Errorf("vp_mem_device_count cannot be greater than %d", vmutils.MaxVPMEMCount)
}

if vpmemCount > 0 && vpmemSize%4096 != 0 {
return nil, nil, nil, fmt.Errorf("vp_mem_size_bytes must be a multiple of 4096")
}

log.G(ctx).WithFields(logrus.Fields{
"vpmemCount": vpmemCount,
"vpmemSizeBytes": vpmemSize,
}).Debug("parsed VPMem configuration")

// Extract the rootfs file name.
rootFsFile := filepath.Base(rootFsFullPath)

// Create VPMem controller configuration
var vpMemController *hcsschema.VirtualPMemController
if vpmemCount > 0 {
// Initialize VPMem controller with specified count and size.
vpMemController = &hcsschema.VirtualPMemController{
MaximumCount: vpmemCount,
MaximumSizeBytes: vpmemSize,
}

// If booting from VHD via VPMem, configure the VPMem device for rootfs
if rootFsFile == vmutils.VhdFile {
vpMemController.Devices = make(map[string]hcsschema.VirtualPMemDevice)

// Determine image format based on file extension.
// filepath.Ext returns the extension with the leading dot (e.g. ".vhdx").
imageFormat := "Vhd1"
if strings.HasSuffix(strings.ToLower(filepath.Ext(rootFsFile)), "vhdx") {
imageFormat = "Vhdx"
}

// Add rootfs VHD as VPMem device 0
vpMemController.Devices["0"] = hcsschema.VirtualPMemDevice{
HostPath: rootFsFullPath,
ReadOnly: true,
ImageFormat: imageFormat,
}

log.G(ctx).WithFields(logrus.Fields{
"device": "0",
"path": rootFsFullPath,
"imageFormat": imageFormat,
}).Debug("configured VPMem device for VHD rootfs boot")
}
return nil, nil, fmt.Errorf("v2 shims do not support vPMem devices")
}

// ===============================Parse SCSI configuration===============================
Expand All @@ -122,9 +73,11 @@ func parseDeviceOptions(
}
}

// If booting from VHD via SCSI (no VPMem), attach the rootfs VHD to SCSI controller 0, LUN 0
// If booting from VHD via SCSI, attach the rootfs VHD to SCSI controller 0, LUN 0
// For confidential Containers, rootFSFile will be DmVerityRootfsPath.
if vpmemCount == 0 && rootFsFile == vmutils.VhdFile {
// Extract the rootfs file name.
rootFsFile := filepath.Base(rootFsFullPath)
if rootFsFile == vmutils.VhdFile {
scsiControllers[guestrequest.ScsiControllerGuids[0]].Attachments["0"] = hcsschema.Attachment{
Type_: "VirtualDisk",
Path: rootFsFullPath,
Expand Down Expand Up @@ -169,14 +122,14 @@ func parseDeviceOptions(
if d := getVPCIDevice(ctx, dev); d != nil {
key := deviceKey{instanceID: d.DeviceInstancePath, functionIndex: d.VirtualFunction}
if _, exists := seen[key]; exists {
return nil, nil, nil, fmt.Errorf("device %s with index %d is specified multiple times", d.DeviceInstancePath, d.VirtualFunction)
return nil, nil, fmt.Errorf("device %s with index %d is specified multiple times", d.DeviceInstancePath, d.VirtualFunction)
}
seen[key] = struct{}{}

// Generate a unique VMBus GUID for each vPCI device.
vmbusGUID, err := guid.NewV4()
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to generate vmbus GUID for device %s: %w", d.DeviceInstancePath, err)
return nil, nil, fmt.Errorf("failed to generate vmbus GUID for device %s: %w", d.DeviceInstancePath, err)
}

vpciDevices[vmbusGUID.String()] = hcsschema.VirtualPciDevice{
Expand All @@ -196,7 +149,7 @@ func parseDeviceOptions(
}

log.G(ctx).Debug("parseDeviceOptions completed successfully")
return vpMemController, scsiControllers, vpciDevices, nil
return scsiControllers, vpciDevices, nil
}

// getVPCIDevice maps a WindowsDevice into the sandbox vPCIDevice format when supported.
Expand Down
2 changes: 1 addition & 1 deletion internal/builder/vm/lcow/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// - Boot Configuration: Kernel, initrd, root filesystem, and boot file paths
// - CPU Configuration: Processor count, limits, and NUMA topology
// - Memory Configuration: Memory size, MMIO gaps, and memory affinity
// - Device Configuration: VPMem devices, vPCI devices, and SCSI controllers
// - Device Configuration: vPCI devices, and SCSI controllers
// - Storage Configuration: Storage QoS settings
// - Confidential Computing: Security policies, SNP settings, and encryption
// - Kernel Arguments: Command line parameters derived from all configuration sources
Expand Down
10 changes: 1 addition & 9 deletions internal/builder/vm/lcow/kernel_args.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ func buildKernelArgs(
annotations map[string]string,
processorCount uint32,
kernelDirect bool,
isVPMem bool,
hasConsole bool,
rootFsFile string,
) (string, error) {
Expand All @@ -41,7 +40,7 @@ func buildKernelArgs(
var args []string

// 1. Root filesystem configuration.
rootfsArgs, err := buildRootfsArgs(ctx, annotations, rootFsFile, kernelDirect, isVPMem)
rootfsArgs, err := buildRootfsArgs(ctx, annotations, rootFsFile, kernelDirect)
if err != nil {
return "", err
}
Expand Down Expand Up @@ -95,12 +94,10 @@ func buildRootfsArgs(
annotations map[string]string,
rootFsFile string,
kernelDirect bool,
isVPMem bool,
) (string, error) {
log.G(ctx).WithFields(logrus.Fields{
"rootFsFile": rootFsFile,
"kernelDirect": kernelDirect,
"isVPMem": isVPMem,
}).Debug("buildRootfsArgs: starting rootfs args construction")

isInitrd := rootFsFile == vmutils.InitrdFile
Expand All @@ -113,11 +110,6 @@ func buildRootfsArgs(

// VHD boot
if isVHD {
// VPMem VHD(X) booting.
if isVPMem {
return "root=/dev/pmem0 ro rootwait init=/init", nil
}

// SCSI VHD booting with dm-verity.
dmVerityMode := oci.ParseAnnotationsBool(ctx, annotations, shimannotations.DmVerityMode, false)
if dmVerityMode {
Expand Down
8 changes: 3 additions & 5 deletions internal/builder/vm/lcow/specs.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ func BuildSandboxConfig(
// ===============================================================================

// This should be done after parsing boot options, as some device options may depend on boot settings (e.g., rootfs path).
vpmemCtrl, scsiCtrl, vpciDevices, err := parseDeviceOptions(
scsiCtrl, vpciDevices, err := parseDeviceOptions(
ctx,
spec.Annotations,
spec.Devices,
Expand Down Expand Up @@ -208,7 +208,6 @@ func BuildSandboxConfig(
spec.Annotations,
cpuConfig.Count,
bootOptions.LinuxKernelDirect != nil, // isKernelDirectBoot
vpmemCtrl != nil, // isVPMem
comPorts != nil, // hasConsole
filepath.Base(rootFsFullPath),
)
Expand Down Expand Up @@ -255,9 +254,8 @@ func BuildSandboxConfig(
},
StorageQoS: storageQOSConfig,
Devices: &hcsschema.Devices{
Scsi: scsiCtrl,
VirtualPMem: vpmemCtrl,
VirtualPci: vpciDevices,
Scsi: scsiCtrl,
VirtualPci: vpciDevices,
HvSocket: &hcsschema.HvSocket2{
HvSocketConfig: hvSocketConfig,
},
Expand Down
Loading
Loading