feat(baremetal-agent): Linux PCIE disk support mdadm soft raid

build/docker/Dockerfile.baremetal-agent

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.3.9-20231219.0
+FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.3.9-20251120.1
 
 MAINTAINER "Zexi Li <[email protected]>"
 

build/docker/Dockerfile.baremetal-base

@@ -1,7 +1,7 @@
 #FROM --platform=linux/amd64 registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-4 as build
 #RUN yum install -y https://iso.yunion.cn/vm-images/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
 #RUN yum install -y http://192.168.23.50:8083/baremetal-pxerom-1.1.0-21092209.x86_64.rpm
-FROM registry.cn-beijing.aliyuncs.com/yunionio/yunionos:v3.10.8-20231215.0 as yunionos
+FROM  registry.cn-beijing.aliyuncs.com/yunionio/yunionos:v4.0.0-20251118.0 as yunionos
 
 FROM centos:8 as grub-stage
 

build/docker/Makefile

@@ -30,7 +30,7 @@ WEBCONSOLE_BASE_VERSION = 20230731.2
 webconsole-base:
 	$(DOCKER_BUILDX)/webconsole-base:$(WEBCONSOLE_BASE_VERSION) -f ./Dockerfile.webconsole-base .
 
-BAREMETAL_BASE_VERSION = v0.3.9-20231219.1
+BAREMETAL_BASE_VERSION = v0.3.9-20251112.1
 
 baremetal-base:
 	$(DOCKER_BUILDX)/baremetal-base:$(BAREMETAL_BASE_VERSION) -f ./Dockerfile.baremetal-base .

pkg/apis/compute/api.go

@@ -312,6 +312,7 @@ type BaremetalDiskConfig struct {
 	RA           *bool   `json:"ra,omitempty"`
 	WT           *bool   `json:"wt,omitempty"`
 	Direct       *bool   `json:"direct,omitempty"`
+	SoftRaidIdx  *int    `json:"soft_raid_idx"`
 }
 
 type RootDiskMatcherSizeMBRange struct {

pkg/apis/compute/baremetal_const.go

@@ -64,6 +64,11 @@ var (
 		DISK_DRIVER_ADAPTECRAID,
 	)
 
+	DISK_DRIVERS_SOFT_RAID = sets.NewString(
+		DISK_DRIVER_LINUX,
+		DISK_DRIVER_PCIE,
+	)
+
 	DISK_DRIVERS = sets.NewString(
 		DISK_DRIVER_LINUX,
 		DISK_DRIVER_PCIE).Union(DISK_DRIVERS_RAID)

pkg/baremetal/manager.go

@@ -56,7 +56,9 @@ import (
 	"yunion.io/x/onecloud/pkg/baremetal/utils/disktool"
 	"yunion.io/x/onecloud/pkg/baremetal/utils/grub"
 	"yunion.io/x/onecloud/pkg/baremetal/utils/ipmitool"
+	raid2 "yunion.io/x/onecloud/pkg/baremetal/utils/raid"
 	raiddrivers "yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
+	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/mdadm"
 	"yunion.io/x/onecloud/pkg/baremetal/utils/uefi"
 	"yunion.io/x/onecloud/pkg/cloudcommon/types"
 	"yunion.io/x/onecloud/pkg/compute/baremetal"
@@ -2671,15 +2673,32 @@ func (s *SBaremetalServer) NewConfigedSSHPartitionTool(term *ssh.Client) (*diskt
 		return nil, fmt.Errorf("CalculateLayout: %v", err)
 	}
 
+	log.Errorf("NewConfigedSSHPartitionTool layouts: %s", jsonutils.Marshal(layouts))
 	diskConfs := baremetal.GroupLayoutResultsByDriverAdapter(layouts)
 	for _, dConf := range diskConfs {
 		driver := dConf.Driver
 		adapter := dConf.Adapter
+		isSoftRaid := baremetal.DISK_DRIVERS_SOFT_RAID.Has(driver)
+
 		raidDrv := raiddrivers.GetDriver(driver, term)
 		if raidDrv != nil {
 			if err := raidDrv.ParsePhyDevs(); err != nil {
 				return nil, fmt.Errorf("RaidDriver %s parse physical devices: %v", raidDrv.GetName(), err)
 			}
+			if isSoftRaid {
+				devs := make([]*baremetal.BaremetalStorage, 0)
+				for _, layout := range layouts {
+					if len(layout.Disks) > 0 && layout.Disks[0].Driver == driver && layout.Disks[0].Adapter == dConf.Adapter {
+						devs = append(devs, layout.Disks...)
+					}
+				}
+
+				log.Infof("SetDevicesForAdapter %v", jsonutils.Marshal(devs))
+				if mdadmDrver, ok := raidDrv.(raid2.IRaidDeviceSetter); ok {
+					mdadmDrver.SetDevicesForAdapter(dConf.Adapter, devs)
+				}
+			}
+
 			if err := raiddrivers.PostBuildRaid(raidDrv, adapter); err != nil {
 				return nil, fmt.Errorf("Build %s raid failed: %v", raidDrv.GetName(), err)
 			}
@@ -2731,11 +2750,27 @@ func (s *SBaremetalServer) DoDiskConfig(term *ssh.Client) (*disktool.SSHPartitio
 	for _, dConf := range diskConfs {
 		driver := dConf.Driver
 		adapter := dConf.Adapter
+		isSoftRaid := baremetal.DISK_DRIVERS_SOFT_RAID.Has(driver)
+
 		raidDrv := raiddrivers.GetDriver(driver, term)
 		if raidDrv != nil {
 			if err := raidDrv.ParsePhyDevs(); err != nil {
 				return nil, fmt.Errorf("RaidDriver %s parse physical devices: %v", raidDrv.GetName(), err)
 			}
+			if isSoftRaid {
+				devs := make([]*baremetal.BaremetalStorage, 0)
+				for _, layout := range layouts {
+					if len(layout.Disks) > 0 && layout.Disks[0].Driver == driver && layout.Disks[0].Adapter == dConf.Adapter {
+						devs = append(devs, layout.Disks...)
+					}
+				}
+
+				log.Infof("SetDevicesForAdapter %v", jsonutils.Marshal(devs))
+				if mdadmDriver, ok := raidDrv.(raid2.IRaidDeviceSetter); ok {
+					mdadmDriver.SetDevicesForAdapter(dConf.Adapter, devs)
+				}
+			}
+
 			if err := raiddrivers.BuildRaid(raidDrv, dConf.Configs, adapter); err != nil {
 				return nil, fmt.Errorf("Build %s raid failed: %v", raidDrv.GetName(), err)
 			}
@@ -2779,6 +2814,10 @@ func (s *SBaremetalServer) DoDiskUnconfig(term *ssh.Client) error {
 }
 
 func (s *SBaremetalServer) DoEraseDisk(term *ssh.Client) error {
+	// soft raid should stop mdadm first
+	if err := mdadm.CleanRaid(term); err != nil {
+		return err
+	}
 	cmd := "/lib/mos/partdestroy.sh"
 	_, err := term.Run(cmd)
 	return err
@@ -2839,6 +2878,7 @@ func (s *SBaremetalServer) DoPartitionDisk(tool *disktool.SSHPartitionTool, term
 	rootImageId := s.GetRootTemplateId()
 	diskOffset := 0
 	rootDisk := tool.GetRootDisk()
+	log.Infof("root disk name %s", rootDisk.GetDevName())
 	if len(rootImageId) > 0 {
 		rootDiskObj := disks[0]
 		rootSize, _ := rootDiskObj.Int("size")

pkg/baremetal/tasks/create.go

@@ -54,13 +54,13 @@ func (self *SBaremetalServerCreateTask) RemoveEFIOSEntry() bool {
 }
 
 func (self *SBaremetalServerCreateTask) DoDeploys(ctx context.Context, term *ssh.Client) (jsonutils.JSONObject, error) {
-	// Build raid
-	tool, err := self.Baremetal.GetServer().DoDiskConfig(term)
-	if err != nil {
+	if err := self.Baremetal.GetServer().DoEraseDisk(term); err != nil {
 		return nil, self.onError(ctx, term, err)
 	}
 	time.Sleep(2 * time.Second)
-	if err := self.Baremetal.GetServer().DoEraseDisk(term); err != nil {
+	// Build raid
+	tool, err := self.Baremetal.GetServer().DoDiskConfig(term)
+	if err != nil {
 		return nil, self.onError(ctx, term, err)
 	}
 	time.Sleep(2 * time.Second)

pkg/baremetal/utils/disktool/disktool.go

@@ -17,6 +17,7 @@ package disktool
 import (
 	"fmt"
 	"math"
+	"strconv"
 	"strings"
 
 	"yunion.io/x/jsonutils"
@@ -229,7 +230,7 @@ type DiskPartitions struct {
 	partitions []*Partition
 }
 
-func newDiskPartitions(driver string, adapter int, raidConfig string, sizeMB int64, blockSize int64, diskType string, tool *PartitionTool) *DiskPartitions {
+func newDiskPartitions(driver string, adapter int, raidConfig string, sizeMB int64, blockSize int64, diskType string, softRaidIdx *int, tool *PartitionTool) *DiskPartitions {
 	ps := new(DiskPartitions)
 	ps.driver = driver
 	ps.adapter = adapter
@@ -239,9 +240,32 @@ func newDiskPartitions(driver string, adapter int, raidConfig string, sizeMB int
 	ps.blockSize = blockSize
 	ps.diskType = diskType
 	ps.partitions = make([]*Partition, 0)
+
+	// soft raid, mdadm
+	if softRaidIdx != nil {
+		ps.GetMdadmInfo(softRaidIdx)
+	}
 	return ps
 }
 
+func (ps *DiskPartitions) GetMdadmInfo(softRaidIdx *int) {
+	devLinkName := fmt.Sprintf("/dev/md/md%d", *softRaidIdx)
+	devLinkNickname := fmt.Sprintf("/dev/md/md%d_0", *softRaidIdx)
+	cmd := fmt.Sprintf("readlink -f $(test -e %s && echo %s || echo %s)", devLinkName, devLinkName, devLinkNickname)
+	out, err := ps.tool.Run(cmd)
+	if err != nil || len(out) == 0 {
+		log.Errorf("failed readlink of %s: %s", devLinkName, err)
+		return
+	}
+
+	ps.dev = strings.TrimSpace(out[0])
+	ps.devName = ps.dev
+	uuid, sectors := ps.tool.GetMdadmUuidAndSector(ps.dev)
+	ps.pciPath = uuid
+	ps.sectors = sectors
+	ps.blockSize = 512
+}
+
 func (p *DiskPartitions) IsRaidDriver() bool {
 	return utils.IsInStringArray(p.driver, []string{
 		baremetal.DISK_DRIVER_MEGARAID,
@@ -324,7 +348,7 @@ func (ps *DiskPartitions) IsReady() bool {
 
 func (ps *DiskPartitions) GetDevName() string {
 	devName := ps.devName
-	if !ps.IsRaidDriver() || ps.raidConfig == baremetal.DISK_CONF_NONE {
+	if ps.raidConfig == baremetal.DISK_CONF_NONE {
 		return devName
 	}
 	raidDrv, err := raiddrivers.GetDriverWithInit(ps.driver, ps.tool.runner.Term())
@@ -690,12 +714,13 @@ func (tool *PartitionTool) parseLsDisk(lines []string, driver string) {
 
 func (tool *PartitionTool) FetchDiskConfs(diskConfs []baremetal.DiskConfiguration) *PartitionTool {
 	for _, d := range diskConfs {
-		disk := newDiskPartitions(d.Driver, d.Adapter, d.RaidConfig, d.Size, d.Block, d.DiskType, tool)
+		disk := newDiskPartitions(d.Driver, d.Adapter, d.RaidConfig, d.Size, d.Block, d.DiskType, d.SoftRaidIdx, tool)
 		tool.disks = append(tool.disks, disk)
+		isSoftRaid := d.RaidConfig != baremetal.DISK_CONF_NONE
 		var key string
-		if d.Driver == baremetal.DISK_DRIVER_LINUX {
+		if d.Driver == baremetal.DISK_DRIVER_LINUX && !isSoftRaid {
 			key = NONRAID_DRIVER
-		} else if d.Driver == baremetal.DISK_DRIVER_PCIE {
+		} else if d.Driver == baremetal.DISK_DRIVER_PCIE && !isSoftRaid {
 			key = PCIE_DRIVER
 		} else {
 			key = RAID_DRVIER
@@ -768,7 +793,42 @@ func (tool *PartitionTool) IsAllDisksReady() bool {
 	return true
 }
 
+func (tool *PartitionTool) GetMdadmUuidAndSector(devPath string) (string, int64) {
+	var uuid string
+	var sectorsRet int64
+	// get md uuid as pci path
+	cmd := fmt.Sprintf("/sbin/mdadm --detail %s | grep UUID", devPath)
+	output, err := tool.Run(cmd)
+	if err == nil && len(output) > 0 {
+		uuidSeg := output[0]
+		segs := strings.SplitN(strings.TrimSpace(uuidSeg), ":", 2)
+		if len(segs) == 2 {
+			uuid = strings.TrimSpace(segs[1])
+		}
+	}
+
+	// get block size
+	cmd = fmt.Sprintf("blockdev --getsz %s 2>/dev/null || echo 0", devPath)
+	output, err = tool.Run(cmd)
+	if err == nil && len(output) > 0 {
+		if sectors, err := strconv.ParseInt(strings.TrimSpace(output[0]), 10, 64); err == nil {
+			sectorsRet = sectors
+		}
+	}
+	return uuid, sectorsRet
+}
+
 func (tool *PartitionTool) RetrieveDiskInfo(rootMatcher *api.BaremetalRootDiskMatcher) error {
+	for _, disk := range tool.disks {
+		if baremetal.DISK_DRIVERS_SOFT_RAID.Has(disk.driver) && disk.raidConfig != baremetal.DISK_CONF_NONE {
+			log.Infof("Soft raid mdadm set diskinfo dev %s", disk.dev)
+			uuid, sectors := tool.GetMdadmUuidAndSector(disk.dev)
+			disk.pciPath = uuid
+			disk.sectors = sectors
+			disk.blockSize = 512
+		}
+	}
+
 	for _, driver := range []string{RAID_DRVIER, NONRAID_DRIVER, PCIE_DRIVER} {
 		cmd := fmt.Sprintf("/lib/mos/lsdisk --%s", driver)
 		ret, err := tool.Run(cmd)

pkg/baremetal/utils/raid/drivers/drivers.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/onecloud/pkg/baremetal/utils/raid"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/adaptec"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/hpssactl"
+	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/mdadm"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/megactl"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/mvcli"
 	_ "yunion.io/x/onecloud/pkg/baremetal/utils/raid/sas2iru"

pkg/baremetal/utils/raid/interface.go

@@ -45,6 +45,10 @@ type IRaidAdapter interface {
 	PostBuildRaid() error
 }
 
+type IRaidDeviceSetter interface {
+	SetDevicesForAdapter(int, []*baremetal.BaremetalStorage)
+}
+
 type IExecTerm interface {
 	Run(cmds ...string) ([]string, error)
 	RunWithInput(input io.Reader, cmds ...string) ([]string, error)

pkg/baremetal/utils/raid/mdadm/doc.go

@@ -0,0 +1 @@
+package mdadm // import "yunion.io/x/onecloud/pkg/baremetal/utils/raid/mdadm"