Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate UPF feature and tests #883

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Integrate UPF feature and tests #883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
bbf695e
Integrate UPF feature and tests
char-1ee Nov 14, 2023
9fd3347
Apply gofmt formatting
char-1ee Nov 22, 2023
7c815fb
Fix UPF legacy
char-1ee Dec 11, 2023
11c9cbd
Clean up formatting
char-1ee Dec 11, 2023
5c2fff5
Update firecracker binaries
char-1ee Dec 12, 2023
0ffd48d
Refactor parameters
char-1ee Dec 19, 2023
065cb9f
Fix mem_backend and mem_file_path exclusion
char-1ee Dec 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,8 @@ SUBDIRS:=ctriface taps misc profile
EXTRAGOARGS:=-v -race -cover
EXTRAGOARGS_NORACE:=-v
EXTRATESTFILES:=vhive_test.go stats.go vhive.go functions.go
# User-level page faults are temporarily disabled (gh-807)
# WITHUPF:=-upfTest
# WITHLAZY:=-lazyTest
WITHUPF:=
WITHLAZY:=
WITHUPF:=-upfTest
WITHLAZY:=-lazyTest
WITHSNAPSHOTS:=-snapshotsTest
CTRDLOGDIR:=/tmp/ctrd-logs

Expand Down
4 changes: 2 additions & 2 deletions bin/containerd-shim-aws-firecracker
View file
Open in desktop
Git LFS file not shown
4 changes: 2 additions & 2 deletions bin/default-rootfs.img
View file
Open in desktop
Git LFS file not shown
2 changes: 1 addition & 1 deletion bin/firecracker
View file
Open in desktop
Git LFS file not shown
4 changes: 2 additions & 2 deletions bin/firecracker-containerd
View file
Open in desktop
Git LFS file not shown
4 changes: 2 additions & 2 deletions bin/firecracker-ctr
View file
Open in desktop
Git LFS file not shown
7 changes: 2 additions & 5 deletions ctriface/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,8 @@
EXTRAGOARGS:=-v -race -cover
EXTRATESTFILES:=iface_test.go iface.go orch_options.go orch.go
BENCHFILES:=bench_test.go iface.go orch_options.go orch.go
# User-level page faults are temporarily disabled (gh-807)
# WITHUPF:=-upf
# WITHLAZY:=-lazy
WITHUPF:=
WITHLAZY:=
WITHUPF:=-upf
WITHLAZY:=-lazy
GOBENCH:=-v -timeout 1500s
CTRDLOGDIR:=/tmp/ctrd-logs

Expand Down
30 changes: 19 additions & 11 deletions ctriface/iface.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ type StartVMResponse struct {

const (
testImageName = "ghcr.io/ease-lab/helloworld:var_workload"
fileBackend = "File"
uffdBackend = "Uffd"
)

// StartVM Boots a VM if it does not exist
Expand Down Expand Up @@ -104,7 +106,7 @@ func (o *Orchestrator) StartVMWithEnvironment(ctx context.Context, vmID, imageNa

tStart = time.Now()
conf := o.getVMConfig(vm)
_, err = o.fcClient.CreateVM(ctx, conf)
resp, err := o.fcClient.CreateVM(ctx, conf)
startVMMetric.MetricMap[metrics.FcCreateVM] = metrics.ToUS(time.Since(tStart))
if err != nil {
return nil, nil, errors.Wrap(err, "failed to create the microVM in firecracker-containerd")
Expand Down Expand Up @@ -206,15 +208,14 @@ func (o *Orchestrator) StartVMWithEnvironment(ctx context.Context, vmID, imageNa
logger.Debug("Registering VM with the memory manager")

stateCfg := manager.SnapshotStateCfg{
VMID: vmID,
GuestMemPath: o.getMemoryFile(vmID),
BaseDir: o.getVMBaseDir(vmID),
GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024,
IsLazyMode: o.isLazyMode,
VMMStatePath: o.getSnapshotFile(vmID),
WorkingSetPath: o.getWorkingSetFile(vmID),
// FIXME (gh-807)
//InstanceSockAddr: resp.UPFSockPath,
VMID: vmID,
GuestMemPath: o.getMemoryFile(vmID),
BaseDir: o.getVMBaseDir(vmID),
GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024,
IsLazyMode: o.isLazyMode,
VMMStatePath: o.getSnapshotFile(vmID),
WorkingSetPath: o.getWorkingSetFile(vmID),
InstanceSockAddr: resp.GetSocketPath(),
}
if err := o.memoryManager.RegisterVM(stateCfg); err != nil {
return nil, nil, errors.Wrap(err, "failed to register VM with memory manager")
Expand Down Expand Up @@ -494,13 +495,20 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap
conf := o.getVMConfig(vm)
conf.LoadSnapshot = true
conf.SnapshotPath = snap.GetSnapshotFilePath()
conf.MemFilePath = snap.GetMemFilePath()
conf.ContainerSnapshotPath = containerSnap.GetDevicePath()

if o.GetUPFEnabled() {
conf.MemBackend.BackendType = uffdBackend
conf.MemBackend.BackendPath, err = o.memoryManager.GetUPFSockPath(vmID)
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to get UPF socket path for uffd backend")
}

if err := o.memoryManager.FetchState(vmID); err != nil {
return nil, nil, err
}
} else {
conf.MemFilePath = snap.GetMemFilePath()
}

tStart = time.Now()
Expand Down
6 changes: 0 additions & 6 deletions ctriface/iface_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,6 @@ var (

func TestMain(m *testing.M) {
char-1ee marked this conversation as resolved.
Show resolved Hide resolved
flag.Parse()

if *isUPFEnabled {
log.Error("User-level page faults are temporarily disabled (gh-807)")
os.Exit(-1)
}

os.Exit(m.Run())
}

Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ replace (
)

replace (
github.com/firecracker-microvm/firecracker-containerd => github.com/vhive-serverless/firecracker-containerd v0.0.0-20230912063208-ad6383f05e45
github.com/firecracker-microvm/firecracker-containerd => github.com/char-1ee/firecracker-containerd v0.0.0-20231018191519-49cac5eea134
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

don't forget to update our fork after testing

github.com/vhive-serverless/vhive/examples/protobuf/helloworld => ./examples/protobuf/helloworld
)

Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,8 @@ github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6Z
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
github.com/char-1ee/firecracker-containerd v0.0.0-20231018191519-49cac5eea134 h1:InrwKCxhDU1PJTNJ0wOHM/PvsIruaz2HriViJ5swrX4=
github.com/char-1ee/firecracker-containerd v0.0.0-20231018191519-49cac5eea134/go.mod h1:XC5a/4PWbzipD5Ron745odZxoVy/J6d8xFldwTZJbSU=
github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho=
github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
Expand Down Expand Up @@ -1022,8 +1024,6 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/vhive-serverless/firecracker-containerd v0.0.0-20230912063208-ad6383f05e45 h1:B+2NmtrRoWgfYkaqqG9Dyqud5HRjfibFpB8wbqER/PQ=
github.com/vhive-serverless/firecracker-containerd v0.0.0-20230912063208-ad6383f05e45/go.mod h1:XC5a/4PWbzipD5Ron745odZxoVy/J6d8xFldwTZJbSU=
github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
Expand Down
29 changes: 29 additions & 0 deletions memory/manager/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,35 @@ func (m *MemoryManager) GetUPFLatencyStats(vmID string) ([]*metrics.Metric, erro
return state.latencyMetrics, nil
}

func (m *MemoryManager) GetUPFSockPath(vmID string) (string, error) {
logger := log.WithFields(log.Fields{"vmID": vmID})

logger.Debug("Get the path of firecracker unix domain socket")

m.Lock()

state, ok := m.instances[vmID]
if !ok {
m.Unlock()
logger.Error("VM not registered with the memory manager")
return "", errors.New("VM not registered with the memory manager")
}

m.Unlock()

if state.isActive {
logger.Error("Cannot get stats while VM is active")
return "", errors.New("Cannot get stats while VM is active")
}

if !m.MetricsModeOn || !state.metricsModeOn {
logger.Error("Metrics mode is not on")
return "", errors.New("Metrics mode is not on")
}

return m.instances[vmID].SnapshotStateCfg.InstanceSockAddr, nil
}

func getLazyHeaderStats(state *SnapshotState, functionName string) ([]string, []string) {
header := []string{
"FuncName",
Expand Down
5 changes: 0 additions & 5 deletions vhive.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,6 @@ func main() {
return
}

if *isUPFEnabled {
log.Error("User-level page faults are temporarily disabled (gh-807)")
return
}

if *isUPFEnabled && !*isSnapshotsEnabled {
log.Error("User-level page faults are not supported without snapshots")
return
Expand Down
5 changes: 0 additions & 5 deletions vhive_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,6 @@ func TestMain(m *testing.M) {

flag.Parse()

if *isUPFEnabledTest {
log.Error("User-level page faults are temporarily disabled (gh-807)")
os.Exit(-1)
}

log.Infof("Orchestrator snapshots enabled: %t", *isSnapshotsEnabledTest)
log.Infof("Orchestrator UPF enabled: %t", *isUPFEnabledTest)
log.Infof("Orchestrator lazy serving mode enabled: %t", *isLazyModeTest)
Expand Down
Loading