|
@@ -11,22 +11,25 @@ import (
|
|
|
"github.com/opencost/opencost/core/pkg/source"
|
|
"github.com/opencost/opencost/core/pkg/source"
|
|
|
)
|
|
)
|
|
|
|
|
|
|
|
-func TestComputeDCGMDevices(t *testing.T) {
|
|
|
|
|
|
|
+func TestComputeDevices(t *testing.T) {
|
|
|
start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
|
end := start.Add(time.Hour)
|
|
end := start.Add(time.Hour)
|
|
|
|
|
|
|
|
tests := []struct {
|
|
tests := []struct {
|
|
|
- name string
|
|
|
|
|
- overrides map[string]any
|
|
|
|
|
- want map[string]*kubemodel.DCGMDevice
|
|
|
|
|
|
|
+ name string
|
|
|
|
|
+ overrides map[string]any
|
|
|
|
|
+ containers map[string]*kubemodel.Container
|
|
|
|
|
+ wantDevices map[string]*kubemodel.Device
|
|
|
|
|
+ wantUsageKey string
|
|
|
|
|
+ wantUsage *kubemodel.DeviceUsage
|
|
|
}{
|
|
}{
|
|
|
{
|
|
{
|
|
|
- name: "no data returns empty dcgm device map",
|
|
|
|
|
- overrides: map[string]any{},
|
|
|
|
|
- want: map[string]*kubemodel.DCGMDevice{},
|
|
|
|
|
|
|
+ name: "no data returns empty device map",
|
|
|
|
|
+ overrides: map[string]any{},
|
|
|
|
|
+ wantDevices: map[string]*kubemodel.Device{},
|
|
|
},
|
|
},
|
|
|
{
|
|
{
|
|
|
- name: "basic dcgm device info and uptime",
|
|
|
|
|
|
|
+ name: "basic device info and uptime",
|
|
|
overrides: map[string]any{
|
|
overrides: map[string]any{
|
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
|
{UUID: "GPU-abc123", Device: "nvidia0", ModelName: "A100"},
|
|
{UUID: "GPU-abc123", Device: "nvidia0", ModelName: "A100"},
|
|
@@ -35,28 +38,27 @@ func TestComputeDCGMDevices(t *testing.T) {
|
|
|
{UUID: "GPU-abc123", First: start, Last: end},
|
|
{UUID: "GPU-abc123", First: start, Last: end},
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
- want: map[string]*kubemodel.DCGMDevice{
|
|
|
|
|
|
|
+ wantDevices: map[string]*kubemodel.Device{
|
|
|
"GPU-abc123": {
|
|
"GPU-abc123": {
|
|
|
UUID: "GPU-abc123",
|
|
UUID: "GPU-abc123",
|
|
|
Device: "nvidia0",
|
|
Device: "nvidia0",
|
|
|
ModelName: "A100",
|
|
ModelName: "A100",
|
|
|
Start: start,
|
|
Start: start,
|
|
|
End: end,
|
|
End: end,
|
|
|
- PodUsages: map[string]kubemodel.DCGMPod{},
|
|
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
{
|
|
{
|
|
|
- name: "dcgm device without uptime is not registered",
|
|
|
|
|
|
|
+ name: "device without uptime is not registered",
|
|
|
overrides: map[string]any{
|
|
overrides: map[string]any{
|
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
|
{UUID: "GPU-abc123", Device: "nvidia0", ModelName: "A100"},
|
|
{UUID: "GPU-abc123", Device: "nvidia0", ModelName: "A100"},
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
- want: map[string]*kubemodel.DCGMDevice{},
|
|
|
|
|
|
|
+ wantDevices: map[string]*kubemodel.Device{},
|
|
|
},
|
|
},
|
|
|
{
|
|
{
|
|
|
- name: "dcgm device with empty uuid is skipped",
|
|
|
|
|
|
|
+ name: "device with empty uuid is skipped",
|
|
|
overrides: map[string]any{
|
|
overrides: map[string]any{
|
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
|
{UUID: "", Device: "nvidia0", ModelName: "A100"},
|
|
{UUID: "", Device: "nvidia0", ModelName: "A100"},
|
|
@@ -65,10 +67,31 @@ func TestComputeDCGMDevices(t *testing.T) {
|
|
|
{UUID: "GPU-abc123", First: start, Last: end},
|
|
{UUID: "GPU-abc123", First: start, Last: end},
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
- want: map[string]*kubemodel.DCGMDevice{},
|
|
|
|
|
|
|
+ wantDevices: map[string]*kubemodel.Device{},
|
|
|
},
|
|
},
|
|
|
{
|
|
{
|
|
|
- name: "dcgm container usage avg and max are populated",
|
|
|
|
|
|
|
+ name: "duplicate device info entries use first occurrence",
|
|
|
|
|
+ overrides: map[string]any{
|
|
|
|
|
+ source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
|
|
|
+ {UUID: "GPU-abc123", Device: "nvidia0", ModelName: "A100"},
|
|
|
|
|
+ {UUID: "GPU-abc123", Device: "nvidia0-dup", ModelName: "A100-dup"},
|
|
|
|
|
+ },
|
|
|
|
|
+ source.QueryDCGMDeviceUptime: []*source.DCGMDeviceUptimeResult{
|
|
|
|
|
+ {UUID: "GPU-abc123", First: start, Last: end},
|
|
|
|
|
+ },
|
|
|
|
|
+ },
|
|
|
|
|
+ wantDevices: map[string]*kubemodel.Device{
|
|
|
|
|
+ "GPU-abc123": {
|
|
|
|
|
+ UUID: "GPU-abc123",
|
|
|
|
|
+ Device: "nvidia0",
|
|
|
|
|
+ ModelName: "A100",
|
|
|
|
|
+ Start: start,
|
|
|
|
|
+ End: end,
|
|
|
|
|
+ },
|
|
|
|
|
+ },
|
|
|
|
|
+ },
|
|
|
|
|
+ {
|
|
|
|
|
+ name: "container usage avg and max are applied to a registered container",
|
|
|
overrides: map[string]any{
|
|
overrides: map[string]any{
|
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
|
{UUID: "GPU-abc123", Device: "nvidia0", ModelName: "A100"},
|
|
{UUID: "GPU-abc123", Device: "nvidia0", ModelName: "A100"},
|
|
@@ -83,22 +106,20 @@ func TestComputeDCGMDevices(t *testing.T) {
|
|
|
{UUID: "GPU-abc123", PodUID: "pod-1", Container: "training", Value: 0.95},
|
|
{UUID: "GPU-abc123", PodUID: "pod-1", Container: "training", Value: 0.95},
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
- want: map[string]*kubemodel.DCGMDevice{
|
|
|
|
|
|
|
+ containers: map[string]*kubemodel.Container{
|
|
|
|
|
+ "pod-1/training": {PodUID: "pod-1", Name: "training"},
|
|
|
|
|
+ },
|
|
|
|
|
+ wantDevices: map[string]*kubemodel.Device{
|
|
|
"GPU-abc123": {
|
|
"GPU-abc123": {
|
|
|
UUID: "GPU-abc123",
|
|
UUID: "GPU-abc123",
|
|
|
Device: "nvidia0",
|
|
Device: "nvidia0",
|
|
|
ModelName: "A100",
|
|
ModelName: "A100",
|
|
|
Start: start,
|
|
Start: start,
|
|
|
End: end,
|
|
End: end,
|
|
|
- PodUsages: map[string]kubemodel.DCGMPod{
|
|
|
|
|
- "pod-1": {
|
|
|
|
|
- ContainerUsages: map[string]kubemodel.DCGMContainer{
|
|
|
|
|
- "training": {UsageAvg: 0.75, UsageMax: 0.95},
|
|
|
|
|
- },
|
|
|
|
|
- },
|
|
|
|
|
- },
|
|
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
|
|
+ wantUsageKey: "pod-1/training",
|
|
|
|
|
+ wantUsage: &kubemodel.DeviceUsage{UsageAvg: 0.75, UsageMax: 0.95},
|
|
|
},
|
|
},
|
|
|
{
|
|
{
|
|
|
name: "usage with empty pod uid or container is ignored",
|
|
name: "usage with empty pod uid or container is ignored",
|
|
@@ -114,36 +135,41 @@ func TestComputeDCGMDevices(t *testing.T) {
|
|
|
{UUID: "GPU-abc123", PodUID: "pod-1", Container: "", Value: 0.5},
|
|
{UUID: "GPU-abc123", PodUID: "pod-1", Container: "", Value: 0.5},
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
- want: map[string]*kubemodel.DCGMDevice{
|
|
|
|
|
|
|
+ containers: map[string]*kubemodel.Container{
|
|
|
|
|
+ "pod-1/training": {PodUID: "pod-1", Name: "training"},
|
|
|
|
|
+ },
|
|
|
|
|
+ wantDevices: map[string]*kubemodel.Device{
|
|
|
"GPU-abc123": {
|
|
"GPU-abc123": {
|
|
|
UUID: "GPU-abc123",
|
|
UUID: "GPU-abc123",
|
|
|
Device: "nvidia0",
|
|
Device: "nvidia0",
|
|
|
ModelName: "A100",
|
|
ModelName: "A100",
|
|
|
Start: start,
|
|
Start: start,
|
|
|
End: end,
|
|
End: end,
|
|
|
- PodUsages: map[string]kubemodel.DCGMPod{},
|
|
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
|
|
+ wantUsageKey: "pod-1/training",
|
|
|
|
|
+ wantUsage: nil,
|
|
|
},
|
|
},
|
|
|
{
|
|
{
|
|
|
- name: "duplicate device info entries use first occurrence",
|
|
|
|
|
|
|
+ name: "usage for an unregistered container is ignored",
|
|
|
overrides: map[string]any{
|
|
overrides: map[string]any{
|
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
source.QueryDCGMDeviceInfo: []*source.DCGMDeviceInfoResult{
|
|
|
{UUID: "GPU-abc123", Device: "nvidia0", ModelName: "A100"},
|
|
{UUID: "GPU-abc123", Device: "nvidia0", ModelName: "A100"},
|
|
|
- {UUID: "GPU-abc123", Device: "nvidia0-dup", ModelName: "A100-dup"},
|
|
|
|
|
},
|
|
},
|
|
|
source.QueryDCGMDeviceUptime: []*source.DCGMDeviceUptimeResult{
|
|
source.QueryDCGMDeviceUptime: []*source.DCGMDeviceUptimeResult{
|
|
|
{UUID: "GPU-abc123", First: start, Last: end},
|
|
{UUID: "GPU-abc123", First: start, Last: end},
|
|
|
},
|
|
},
|
|
|
|
|
+ source.QueryDCGMContainerUsageAvg: []*source.DCGMDeviceContainerUsageResult{
|
|
|
|
|
+ {UUID: "GPU-abc123", PodUID: "pod-1", Container: "training", Value: 0.75},
|
|
|
|
|
+ },
|
|
|
},
|
|
},
|
|
|
- want: map[string]*kubemodel.DCGMDevice{
|
|
|
|
|
|
|
+ wantDevices: map[string]*kubemodel.Device{
|
|
|
"GPU-abc123": {
|
|
"GPU-abc123": {
|
|
|
UUID: "GPU-abc123",
|
|
UUID: "GPU-abc123",
|
|
|
Device: "nvidia0",
|
|
Device: "nvidia0",
|
|
|
ModelName: "A100",
|
|
ModelName: "A100",
|
|
|
Start: start,
|
|
Start: start,
|
|
|
End: end,
|
|
End: end,
|
|
|
- PodUsages: map[string]kubemodel.DCGMPod{},
|
|
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
|
},
|
|
},
|
|
@@ -162,11 +188,25 @@ func TestComputeDCGMDevices(t *testing.T) {
|
|
|
require.NoError(t, err)
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
kms := kubemodel.NewKubeModelSet(start, end)
|
|
kms := kubemodel.NewKubeModelSet(start, end)
|
|
|
|
|
+ if tt.containers != nil {
|
|
|
|
|
+ kms.Containers = tt.containers
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
- err = km.computeDCGMDevices(kms, start, end)
|
|
|
|
|
|
|
+ err = km.computeDevices(kms, start, end)
|
|
|
require.NoError(t, err)
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
- assert.Equal(t, tt.want, kms.DCGMDevices)
|
|
|
|
|
|
|
+ assert.Equal(t, tt.wantDevices, kms.Devices)
|
|
|
|
|
+
|
|
|
|
|
+ if tt.wantUsageKey != "" {
|
|
|
|
|
+ c, ok := kms.Containers[tt.wantUsageKey]
|
|
|
|
|
+ require.True(t, ok)
|
|
|
|
|
+ if tt.wantUsage == nil {
|
|
|
|
|
+ assert.Empty(t, c.DeviceUsages)
|
|
|
|
|
+ } else {
|
|
|
|
|
+ require.NotNil(t, c.DeviceUsages)
|
|
|
|
|
+ assert.Equal(t, *tt.wantUsage, c.DeviceUsages["GPU-abc123"])
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
})
|
|
})
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|