Hi there!

Today we will make a script for automatic resizing of EBS volume using golang and AWS SDK version 2. To do this, I need to perform a few steps: get a list of volumes, filter those volumes in which the amount of free memory is less than the threshold, find the ebs id that corresponds to this volume. Resize by a specified percentage and increase the disk size on the file system.

I’ll start with the structure in which all the information on the disk will be stored and specify all the necessary imports.

package main

import (
	"context"
	"errors"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/config"
	"github.com/aws/aws-sdk-go-v2/service/ec2"
	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
	"github.com/aws/smithy-go"
	"github.com/mvisonneau/go-ebsnvme/pkg/ebsnvme"
	"github.com/shirou/gopsutil/v3/disk"
)

type DiskData struct {
	VolumeID   string
	DeviceName string
	MountPoint string
	TotalUsed  float64
	TotalSpace uint64
	FsType     string
	VolumeSize int32
}
}

The first method will write all the necessary information about the disks in the DiskData structure. Gopsutil will be used to retrieve information from the system. If in a system the disk is displayed as nvme it needs to be brought to standard record (/dev/sda) which can be used to filter the necessary ebs volumes. But this alone will not be enough because disks on different servers can be mounted under the same name. To prevent modification of the wrong volume an instance id will be used as an additional condition in a filter. It can be obtained from the server metadata.

First I get a list of all disks using disk.Partitions(false), also I create ec2client and get instanceID from metadata. These methods will be created later. Next, in cycle, i loop through all the disks and check if nvme is in the disk name. If yes with ebsnvme i get disk name in format /dev/sda. And save the result in ebsDevice. If not that I change a disk name from xvd to sd. This is the format required for AWS. Now I can query aws to get the volumeID. Using the Usage method from AWS SDK. I get disk usage and write it to the DiskData.

func filterDisks() ([]DiskData, error) {
	parts, err := disk.Partitions(false)
	if err != nil {
		return nil, err
	}
	client, err := getEc2Client()
	if err != nil {
		return nil, err
	}

	instanceID, err := getInstanceID()
	if err != nil {
		return nil, err
	}
	diskData := []DiskData{}

	for _, p := range parts {
		var ebsDevice string

		if strings.Contains(p.Device, "nvme") {
			volumeMapping, err := ebsnvme.ScanDevice(p.Device)
			if err != nil {
				return nil, err
			}
			ebsDevice = volumeMapping.Name
		} else {
			ebsDevice = strings.Replace(p.Device, "xvd", "sd", 1)
		}

		filter := &ec2.DescribeVolumesInput{Filters: []types.Filter{
			{
				Name: aws.String("attachment.device"),
				Values: []string{
					ebsDevice,
				},
			},
			{
				Name: aws.String("attachment.instance-id"),
				Values: []string{
					instanceID,
				},
			},
		},
		}

		volumeInfo, err := client.DescribeVolumes(context.Background(), filter)
		if err != nil {
			return nil, err
		}
		usage, _ := disk.Usage(p.Mountpoint)
		disk := DiskData{
			VolumeID:   *volumeInfo.Volumes[0].VolumeId,
			DeviceName: p.Device,
			MountPoint: p.Mountpoint,
			TotalUsed:  usage.UsedPercent,
			TotalSpace: usage.Total,
			VolumeSize: volumeInfo.Volumes[0].Size,
			FsType:     p.Fstype,
		}
		diskData = append(diskData, disk)
	}
	return diskData, nil
}

I need to create the methods used by filterDisks, starting from getInstanceID. This method queries the ec2 instance metadata and receives an InstanceID in response.

func getInstanceID() (string, error) {
	cfg, err := config.LoadDefaultConfig(context.TODO())
	if err != nil {
		return "", err
	}

	client := imds.NewFromConfig(cfg)
	instanceID, err := client.GetInstanceIdentityDocument(context.TODO(), &imds.GetInstanceIdentityDocumentInput{})
	if err != nil {
		return "", err
	}
	return instanceID.InstanceID, nil
}

getEc2Client gets the region name from the metadata and returns the ec2 client.

func getEc2Client() (*ec2.Client, error) {
	cfg, err := config.LoadDefaultConfig(context.TODO())
	if err != nil {
		return nil, err
	}

	client := imds.NewFromConfig(cfg)
	region, err := client.GetRegion(context.TODO(), &imds.GetRegionInput{})
	if err != nil {
		return nil, err
	}

	cfg.Region = region.Region
	return ec2.NewFromConfig(cfg), err
}

What is left is to calculate what amounts of gigabytes disk should be resized. This method converts the disk size obtained in filterDisks from bytes to gigabytes and determines new size of ebs volume.

func findNewSize(oldSize uint64, increasePercent float64) int32 {
	gbSize := float64(oldSize) / math.Pow(1024, 3)
	newSize := ((gbSize * increasePercent) / 100) + gbSize
	return int32(newSize)
}

The resize will take place in three steps:

  1. Increase the size of the ebs

  2. Call the grow part to increase the size of the particle if it exists.

  3. Increase the size of the file system.

I’ll start with the first step. In this method, I get the ec2 client using the getEc2Client method and make a ModifyVolume request to AWS. Here it is important to filter out certain errors. The first is VolumeModificationRateExceeded. AWS allows you to resize once every 6 hours, so I don’t want the script to fail if the limit is exceeded.

The next error is IncorrectModificationState. After resizing the disk, its status changes to Optimizing, in which case I also do not want to quit, and go to the next disk.

func ebsResize(newSize int32, volumeID string) error {
	client, err := getEc2Client()
	if err != nil {
		return err
	}
	log.Println("Starting resize of ebs volume", volumeID)
	input := &ec2.ModifyVolumeInput{VolumeId: &volumeID, Size: newSize}
	if _, err := client.ModifyVolume(context.Background(), input); err != nil {
		var ae smithy.APIError
		if errors.As(err, &ae) {
			switch ae.ErrorCode() {
			case "VolumeModificationRateExceeded":
				log.Println("Ebs was already resized, wait for 6 hours before next resize")
				return errVolumeRetryLater
			case "IncorrectModificationState":
				log.Println(ae.ErrorMessage())
				return errVolumeRetryLater
			}
		} else {
			return err
		}
	}
	err = waitForEbsResize(volumeID)
	if err != nil {
		return err
	}
	return nil
}

Once the resize is started, I need to wait for it to complete, so I will create the following method waitForEbsResize. The waitForEbsResize method uses the ec2 client to make a request of type DescribeVolumesModifications and checks whether the status of the disk is modifying. If yes, it waits for 15 seconds and runs the method recursively again.

func waitForEbsResize(volumeID string) error {
	client, err := getEc2Client()
	if err != nil {
		return err
	}
	input := &ec2.DescribeVolumesModificationsInput{VolumeIds: []string{volumeID}}
	status, err := client.DescribeVolumesModifications(context.Background(), input)

	if status.VolumesModifications[0].ModificationState == "modifying" {
		log.Println("Ebs modification in progress. Waiting for 15 second")
		time.Sleep(15 * time.Second)
		if err := waitForEbsResize(volumeID); err != nil {
			return err
		}
	}
	return nil
}

Now I can proceed to step 2 growPartition.

I need to determine if there is a partition, if there is no partition grow part doesn’t need to be executed. Depending on what type of disk I will determine whether there are partitions or no. Several checks are required. The first check if there are no numbers in the device name. If they are not present then it is not partition and it is not required to do grow part. Next check for nvme devices. If there is a symbol p in the name, then it is a partition. For example /dev/nvme1n1 - disk and /dev/nvme0n1p1 - partition. And the last case if there is an xvd in the name of the device, this check only to filter usual ebs devices with numbers in the name. And depending on each case the growpart is executed. For ebs - growpart /dev/xvdf 1 and for nvme - growpart /dev/nvme0n1 1.

func growPartition(partition string) error {
	log.Println("Starting growpart for", partition)
	var cmd *exec.Cmd
	i := strings.LastIndex(partition, "p")
	isLetter := regexp.MustCompile(`^/dev/+[a-zA-Z]+$`).MatchString
	if isLetter(partition) {
		log.Println("Grow partition for", partition, "not required")
	} else if i != -1 {
		cmd = exec.Command("growpart", partition[:i], partition[i+1:])
		return cmd.Run()
	} else if strings.Contains(partition, "xvd") {
		re := regexp.MustCompile(`\D+`)
		m := re.FindString(partition)
		cmd = exec.Command("growpart", m, partition[len(m):])
		return cmd.Run()
	}
	return nil
}

The last step is to increase the size of the file system. How the resize will take place depends on the file system. If it is xfs then called xfs_growfs in other cases it is resize2fs.

func fsResize(filesystem string, mountPoint string, partition string) error {
	log.Println("Starting system volume resize for", partition)
	var cmd *exec.Cmd
	if filesystem == "xfs" {
		cmd = exec.Command("xfs_growfs", "-d", mountPoint)
	} else {
		cmd = exec.Command("resize2fs", partition)
	}
	return cmd.Run()
}

Now it remains to call all the methods in the correct order:

  1. Get a list of disks

  2. Loop in a cycle on all of disks. If the condition disk.TotalUsed < 70 is false then resize not required.

  3. Calculate the new disk size

  4. Call ebsResize

  5. Call growPartition

  6. Call fsResize

func main() {
	disksInfo, err := filterDisks()
	if err != nil {
		log.Fatalln(err)
	}
	for _, disk := range disksInfo {
		if disk.TotalUsed < 70 {
			log.Println("Resize for", disk.DeviceName, "not required")
			continue
		}

		log.Println("Starting resize of", disk.DeviceName)
		newSize := findNewSize(disk.TotalSpace, 30)
		if err := ebsResize(int32(newSize), disk.VolumeID); err == errVolumeRetryLater {
			continue
		} else if err != nil {
			log.Fatalln(err)
		}
		if err := growPartition(disk.DeviceName); err != nil {
			log.Fatalln(err)
		}
		if err := fsResize(disk.FsType, disk.MountPoint, disk.DeviceName); err != nil {
			log.Fatalln(err)
		}
	}
}

After a few seconds, the disk has a new size.