diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index e95536d..62621de 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -252,6 +252,11 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Code: "name_conflict", Message: err.Error(), }, nil + case errors.Is(err, instances.ErrInsufficientResources): + return oapi.CreateInstance409JSONResponse{ + Code: "insufficient_resources", + Message: err.Error(), + }, nil default: log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{ @@ -309,15 +314,15 @@ func (s *ApiService) GetInstanceStats(ctx context.Context, request oapi.GetInsta // vmStatsToOAPI converts vm_metrics.VMStats to oapi.InstanceStats func vmStatsToOAPI(s *vm_metrics.VMStats) oapi.InstanceStats { stats := oapi.InstanceStats{ - InstanceId: s.InstanceID, - InstanceName: s.InstanceName, - CpuSeconds: s.CPUSeconds(), - MemoryRssBytes: int64(s.MemoryRSSBytes), - MemoryVmsBytes: int64(s.MemoryVMSBytes), - NetworkRxBytes: int64(s.NetRxBytes), - NetworkTxBytes: int64(s.NetTxBytes), - AllocatedVcpus: s.AllocatedVcpus, - AllocatedMemoryBytes: s.AllocatedMemoryBytes, + InstanceId: s.InstanceID, + InstanceName: s.InstanceName, + CpuSeconds: s.CPUSeconds(), + MemoryRssBytes: int64(s.MemoryRSSBytes), + MemoryVmsBytes: int64(s.MemoryVMSBytes), + NetworkRxBytes: int64(s.NetRxBytes), + NetworkTxBytes: int64(s.NetTxBytes), + AllocatedVcpus: s.AllocatedVcpus, + AllocatedMemoryBytes: s.AllocatedMemoryBytes, MemoryUtilizationRatio: s.MemoryUtilizationRatio(), } return stats @@ -464,6 +469,11 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan Code: "invalid_state", Message: err.Error(), }, nil + case errors.Is(err, instances.ErrInsufficientResources): + return oapi.StartInstance409JSONResponse{ + Code: "insufficient_resources", + Message: err.Error(), + }, nil default: log.ErrorContext(ctx, "failed to start instance", "error", err) return oapi.StartInstance500JSONResponse{ diff --git a/cmd/api/api/resources.go b/cmd/api/api/resources.go index 6df9328..ebb6ad9 100644 --- a/cmd/api/api/resources.go +++ b/cmd/api/api/resources.go @@ -25,11 +25,13 @@ func (s *ApiService) GetResources(ctx context.Context, _ oapi.GetResourcesReques } // Convert to API response + diskIO := convertResourceStatus(status.DiskIO) resp := oapi.Resources{ Cpu: convertResourceStatus(status.CPU), Memory: convertResourceStatus(status.Memory), Disk: convertResourceStatus(status.Disk), Network: convertResourceStatus(status.Network), + DiskIo: &diskIO, Allocations: make([]oapi.ResourceAllocation, 0, len(status.Allocations)), } @@ -53,6 +55,7 @@ func (s *ApiService) GetResources(ctx context.Context, _ oapi.GetResourcesReques DiskBytes: &alloc.DiskBytes, NetworkDownloadBps: &alloc.NetworkDownloadBps, NetworkUploadBps: &alloc.NetworkUploadBps, + DiskIoBps: &alloc.DiskIOBps, }) } diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 0fd01f1..79dcd0e 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -69,8 +69,7 @@ type Config struct { MaxMemoryPerInstance string // Max memory for a single VM (0 = unlimited) // Resource limits - aggregate - MaxTotalVcpus int // Aggregate vCPU limit across all instances (0 = unlimited) - MaxTotalMemory string // Aggregate memory limit across all instances (0 = unlimited) + // Note: CPU/memory aggregate limits are now handled via oversubscription ratios (OVERSUB_CPU, OVERSUB_MEMORY) MaxTotalVolumeStorage string // Total volume storage limit (0 = unlimited) // OpenTelemetry configuration @@ -166,9 +165,8 @@ func Load() *Config { MaxVcpusPerInstance: getEnvInt("MAX_VCPUS_PER_INSTANCE", 16), MaxMemoryPerInstance: getEnv("MAX_MEMORY_PER_INSTANCE", "32GB"), - // Resource limits - aggregate (0 or empty = unlimited) - MaxTotalVcpus: getEnvInt("MAX_TOTAL_VCPUS", 0), - MaxTotalMemory: getEnv("MAX_TOTAL_MEMORY", ""), + // Resource limits - aggregate + // Note: CPU/memory aggregate limits are now handled via oversubscription ratios MaxTotalVolumeStorage: getEnv("MAX_TOTAL_VOLUME_STORAGE", ""), // OpenTelemetry configuration diff --git a/cmd/api/main.go b/cmd/api/main.go index fef9f32..7f5e426 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -225,6 +225,12 @@ func run() error { logger.Warn("failed to reconcile mdev devices", "error", err) } + // Wire up resource validator for aggregate limit checking + // This enables the instance manager to validate CPU, memory, network, and GPU + // availability before creating or starting instances. + app.InstanceManager.SetResourceValidator(app.ResourceManager) + logger.Info("Resource validator configured") + // Initialize ingress manager (starts Caddy daemon and DNS server for dynamic upstreams) logger.Info("Initializing ingress manager...") if err := app.IngressManager.Initialize(app.Ctx); err != nil { diff --git a/integration/systemd_test.go b/integration/systemd_test.go index bba12df..79973d7 100644 --- a/integration/systemd_test.go +++ b/integration/systemd_test.go @@ -67,8 +67,6 @@ func TestSystemdMode(t *testing.T) { MaxOverlaySize: 100 * 1024 * 1024 * 1024, MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, - MaxTotalVcpus: 0, - MaxTotalMemory: 0, } instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index ea0d668..8c02eab 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -76,8 +76,6 @@ func TestVGPU(t *testing.T) { MaxOverlaySize: 100 * 1024 * 1024 * 1024, MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, - MaxTotalVcpus: 0, - MaxTotalMemory: 0, } instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) @@ -272,4 +270,3 @@ func checkVGPUTestPrerequisites() (string, string) { return "vGPU test requires at least one available VF (all VFs are in use)", "" } - diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 7357ac8..5a9e82c 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -126,6 +126,10 @@ func (m *mockInstanceManager) ListRunningInstancesInfo(ctx context.Context) ([]r return nil, nil } +func (m *mockInstanceManager) SetResourceValidator(v instances.ResourceValidator) { + // no-op for mock +} + // mockVolumeManager implements volumes.Manager for testing type mockVolumeManager struct { volumes map[string]*volumes.Volume @@ -611,7 +615,7 @@ func TestBuildQueue_ConcurrencyLimit(t *testing.T) { queue := NewBuildQueue(2) // Max 2 concurrent started := make(chan string, 5) - + // Enqueue 5 builds with blocking start functions for i := 0; i < 5; i++ { id := string(rune('A' + i)) diff --git a/lib/instances/create.go b/lib/instances/create.go index 904e589..a28d0e2 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -8,7 +8,6 @@ import ( "strings" "time" - "github.com/nrednav/cuid2" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" @@ -16,6 +15,7 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/system" "github.com/kernel/hypeman/lib/volumes" + "github.com/nrednav/cuid2" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "gvisor.dev/gvisor/pkg/cleanup" @@ -48,31 +48,6 @@ var systemDirectories = []string{ "/var", } -// AggregateUsage represents total resource usage across all instances -type AggregateUsage struct { - TotalVcpus int - TotalMemory int64 // in bytes -} - -// calculateAggregateUsage calculates total resource usage across all running instances -func (m *manager) calculateAggregateUsage(ctx context.Context) (AggregateUsage, error) { - instances, err := m.listInstances(ctx) - if err != nil { - return AggregateUsage{}, err - } - - var usage AggregateUsage - for _, inst := range instances { - // Only count running/paused instances (those consuming resources) - if inst.State == StateRunning || inst.State == StatePaused || inst.State == StateCreated { - usage.TotalVcpus += inst.Vcpus - usage.TotalMemory += inst.Size + inst.HotplugSize - } - } - - return usage, nil -} - // generateVsockCID converts first 8 chars of instance ID to a unique CID // CIDs 0-2 are reserved (hypervisor, loopback, host) // Returns value in range 3 to 4294967295 @@ -174,18 +149,12 @@ func (m *manager) createInstance( return nil, fmt.Errorf("total memory %d (size + hotplug_size) exceeds maximum allowed %d per instance", totalMemory, m.limits.MaxMemoryPerInstance) } - // Validate aggregate resource limits - if m.limits.MaxTotalVcpus > 0 || m.limits.MaxTotalMemory > 0 { - usage, err := m.calculateAggregateUsage(ctx) - if err != nil { - log.WarnContext(ctx, "failed to calculate aggregate usage, skipping limit check", "error", err) - } else { - if m.limits.MaxTotalVcpus > 0 && usage.TotalVcpus+vcpus > m.limits.MaxTotalVcpus { - return nil, fmt.Errorf("total vcpus would be %d, exceeds aggregate limit of %d", usage.TotalVcpus+vcpus, m.limits.MaxTotalVcpus) - } - if m.limits.MaxTotalMemory > 0 && usage.TotalMemory+totalMemory > m.limits.MaxTotalMemory { - return nil, fmt.Errorf("total memory would be %d, exceeds aggregate limit of %d", usage.TotalMemory+totalMemory, m.limits.MaxTotalMemory) - } + // Validate aggregate resource limits via ResourceValidator (if configured) + if m.resourceValidator != nil { + needsGPU := req.GPU != nil && req.GPU.Profile != "" + if err := m.resourceValidator.ValidateAllocation(ctx, vcpus, totalMemory, req.NetworkBandwidthDownload, req.NetworkBandwidthUpload, req.DiskIOBps, needsGPU); err != nil { + log.ErrorContext(ctx, "resource validation failed", "error", err) + return nil, fmt.Errorf("%w: %v", ErrInsufficientResources, err) } } diff --git a/lib/instances/errors.go b/lib/instances/errors.go index 1090104..9925bb0 100644 --- a/lib/instances/errors.go +++ b/lib/instances/errors.go @@ -17,4 +17,7 @@ var ( // ErrAmbiguousName is returned when multiple instances have the same name ErrAmbiguousName = errors.New("multiple instances with the same name") + + // ErrInsufficientResources is returned when resources (CPU, memory, network, GPU) are not available + ErrInsufficientResources = errors.New("insufficient resources") ) diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 09c9616..8411d19 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -41,6 +41,9 @@ type Manager interface { // ListRunningInstancesInfo returns info needed for utilization metrics collection. // Used by the resource manager for VM utilization tracking. ListRunningInstancesInfo(ctx context.Context) ([]resources.InstanceUtilizationInfo, error) + // SetResourceValidator sets the validator for aggregate resource limit checking. + // Called after initialization to avoid circular dependencies. + SetResourceValidator(v ResourceValidator) } // ResourceLimits contains configurable resource limits for instances @@ -48,21 +51,28 @@ type ResourceLimits struct { MaxOverlaySize int64 // Maximum overlay disk size in bytes per instance MaxVcpusPerInstance int // Maximum vCPUs per instance (0 = unlimited) MaxMemoryPerInstance int64 // Maximum memory in bytes per instance (0 = unlimited) - MaxTotalVcpus int // Maximum total vCPUs across all instances (0 = unlimited) - MaxTotalMemory int64 // Maximum total memory in bytes across all instances (0 = unlimited) +} + +// ResourceValidator validates if resources can be allocated +type ResourceValidator interface { + // ValidateAllocation checks if the requested resources are available. + // Returns nil if allocation is allowed, or a detailed error describing + // which resource is insufficient and the current capacity/usage. + ValidateAllocation(ctx context.Context, vcpus int, memoryBytes int64, networkDownloadBps int64, networkUploadBps int64, diskIOBps int64, needsGPU bool) error } type manager struct { - paths *paths.Paths - imageManager images.Manager - systemManager system.Manager - networkManager network.Manager - deviceManager devices.Manager - volumeManager volumes.Manager - limits ResourceLimits - instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks - hostTopology *HostTopology // Cached host CPU topology - metrics *Metrics + paths *paths.Paths + imageManager images.Manager + systemManager system.Manager + networkManager network.Manager + deviceManager devices.Manager + volumeManager volumes.Manager + limits ResourceLimits + resourceValidator ResourceValidator // Optional validator for aggregate resource limits + instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks + hostTopology *HostTopology // Cached host CPU topology + metrics *Metrics // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -106,6 +116,12 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste return m } +// SetResourceValidator sets the resource validator for aggregate limit checking. +// This is called after initialization to avoid circular dependencies. +func (m *manager) SetResourceValidator(v ResourceValidator) { + m.resourceValidator = v +} + // getHypervisor creates a hypervisor client for the given socket and type. // Used for connecting to already-running VMs (e.g., for state queries). func (m *manager) getHypervisor(socketPath string, hvType hypervisor.Type) (hypervisor.Hypervisor, error) { @@ -324,6 +340,7 @@ func (m *manager) ListInstanceAllocations(ctx context.Context) ([]resources.Inst VolumeOverlayBytes: volumeOverlayBytes, NetworkDownloadBps: inst.NetworkBandwidthDownload, NetworkUploadBps: inst.NetworkBandwidthUpload, + DiskIOBps: inst.DiskIOBps, State: string(inst.State), VolumeBytes: volumeBytes, }) @@ -366,4 +383,3 @@ func (m *manager) ListRunningInstancesInfo(ctx context.Context) ([]resources.Ins return infos, nil } - diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 839f2b9..4bfb9b4 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -24,6 +24,7 @@ import ( "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/resources" "github.com/kernel/hypeman/lib/system" "github.com/kernel/hypeman/lib/vmm" "github.com/kernel/hypeman/lib/volumes" @@ -54,11 +55,18 @@ func setupTestManager(t *testing.T) (*manager, string) { MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited - MaxTotalVcpus: 0, // unlimited - MaxTotalMemory: 0, // unlimited } mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + // Set up resource validation using the real ResourceManager + resourceMgr := resources.NewManager(cfg, p) + resourceMgr.SetInstanceLister(mgr) + resourceMgr.SetImageLister(imageManager) + resourceMgr.SetVolumeLister(volumeManager) + err = resourceMgr.Initialize(context.Background()) + require.NoError(t, err) + mgr.SetResourceValidator(resourceMgr) + // Register cleanup to kill any orphaned Cloud Hypervisor processes t.Cleanup(func() { cleanupOrphanedProcesses(t, mgr) @@ -922,10 +930,14 @@ func TestStorageOperations(t *testing.T) { tmpDir := t.TempDir() cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", + DataDir: tmpDir, + BridgeName: "vmbr0", + SubnetCIDR: "10.100.0.0/16", + DNSServer: "1.1.1.1", + OversubCPU: 1.0, + OversubMemory: 1.0, + OversubDisk: 1.0, + OversubNetwork: 1.0, } p := paths.New(tmpDir) @@ -938,8 +950,6 @@ func TestStorageOperations(t *testing.T) { MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited - MaxTotalVcpus: 0, // unlimited - MaxTotalMemory: 0, // unlimited } manager := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index 6ff8118..4f34384 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -23,6 +23,7 @@ import ( "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/resources" "github.com/kernel/hypeman/lib/system" "github.com/kernel/hypeman/lib/volumes" "github.com/stretchr/testify/assert" @@ -52,11 +53,18 @@ func setupTestManagerForQEMU(t *testing.T) (*manager, string) { MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited - MaxTotalVcpus: 0, // unlimited - MaxTotalMemory: 0, // unlimited } mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeQEMU, nil, nil).(*manager) + // Set up resource validation using the real ResourceManager + resourceMgr := resources.NewManager(cfg, p) + resourceMgr.SetInstanceLister(mgr) + resourceMgr.SetImageLister(imageManager) + resourceMgr.SetVolumeLister(volumeManager) + err = resourceMgr.Initialize(context.Background()) + require.NoError(t, err) + mgr.SetResourceValidator(resourceMgr) + // Register cleanup to kill any orphaned QEMU processes t.Cleanup(func() { cleanupOrphanedQEMUProcesses(t, mgr) diff --git a/lib/instances/resource_limits_test.go b/lib/instances/resource_limits_test.go index 767d9e7..5b50f1a 100644 --- a/lib/instances/resource_limits_test.go +++ b/lib/instances/resource_limits_test.go @@ -2,10 +2,8 @@ package instances import ( "context" - "os" "syscall" "testing" - "time" "github.com/kernel/hypeman/cmd/api/config" "github.com/kernel/hypeman/lib/devices" @@ -152,7 +150,13 @@ func TestValidateVolumeAttachments_OverlayCountsAsTwoDevices(t *testing.T) { func createTestManager(t *testing.T, limits ResourceLimits) *manager { t.Helper() tmpDir := t.TempDir() - cfg := &config.Config{DataDir: tmpDir} + cfg := &config.Config{ + DataDir: tmpDir, + OversubCPU: 1.0, + OversubMemory: 1.0, + OversubDisk: 1.0, + OversubNetwork: 1.0, + } p := paths.New(cfg.DataDir) imageMgr, err := images.NewManager(p, 1, nil) @@ -171,15 +175,11 @@ func TestResourceLimits_StructValues(t *testing.T) { MaxOverlaySize: 10 * 1024 * 1024 * 1024, // 10GB MaxVcpusPerInstance: 4, MaxMemoryPerInstance: 8 * 1024 * 1024 * 1024, // 8GB - MaxTotalVcpus: 16, - MaxTotalMemory: 32 * 1024 * 1024 * 1024, // 32GB } assert.Equal(t, int64(10*1024*1024*1024), limits.MaxOverlaySize) assert.Equal(t, 4, limits.MaxVcpusPerInstance) assert.Equal(t, int64(8*1024*1024*1024), limits.MaxMemoryPerInstance) - assert.Equal(t, 16, limits.MaxTotalVcpus) - assert.Equal(t, int64(32*1024*1024*1024), limits.MaxTotalMemory) } func TestResourceLimits_ZeroMeansUnlimited(t *testing.T) { @@ -188,8 +188,6 @@ func TestResourceLimits_ZeroMeansUnlimited(t *testing.T) { MaxOverlaySize: 100 * 1024 * 1024 * 1024, MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited - MaxTotalVcpus: 0, // unlimited - MaxTotalMemory: 0, // unlimited } mgr := createTestManager(t, limits) @@ -200,166 +198,8 @@ func TestResourceLimits_ZeroMeansUnlimited(t *testing.T) { assert.Equal(t, int64(0), mgr.limits.MaxMemoryPerInstance) } -func TestAggregateUsage_NoInstances(t *testing.T) { - limits := ResourceLimits{ - MaxOverlaySize: 100 * 1024 * 1024 * 1024, - } - mgr := createTestManager(t, limits) - - usage, err := mgr.calculateAggregateUsage(context.Background()) - require.NoError(t, err) - - assert.Equal(t, 0, usage.TotalVcpus) - assert.Equal(t, int64(0), usage.TotalMemory) -} - -func TestAggregateUsage_StructValues(t *testing.T) { - usage := AggregateUsage{ - TotalVcpus: 8, - TotalMemory: 16 * 1024 * 1024 * 1024, - } - - assert.Equal(t, 8, usage.TotalVcpus) - assert.Equal(t, int64(16*1024*1024*1024), usage.TotalMemory) -} - -// TestAggregateLimits_EnforcedAtRuntime is an integration test that verifies -// aggregate resource limits are enforced when creating VMs. -// It creates one VM, then tries to create another that would exceed the total limit. -func TestAggregateLimits_EnforcedAtRuntime(t *testing.T) { - // Skip in short mode - this is an integration test - if testing.Short() { - t.Skip("skipping integration test in short mode") - } - - // Require KVM access - if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { - t.Skip("/dev/kvm not available - skipping VM test") - } - - ctx := context.Background() - tmpDir := t.TempDir() - - cfg := &config.Config{ - DataDir: tmpDir, - BridgeName: "vmbr0", - SubnetCIDR: "10.100.0.0/16", - DNSServer: "1.1.1.1", - } - - p := paths.New(tmpDir) - imageManager, err := images.NewManager(p, 1, nil) - require.NoError(t, err) - - systemManager := system.NewManager(p) - networkManager := network.NewManager(p, cfg, nil) - deviceManager := devices.NewManager(p) - volumeManager := volumes.NewManager(p, 0, nil) - - // Set small aggregate limits: - // - MaxTotalVcpus: 2 (first VM gets 1, second wants 2 -> denied) - // - MaxTotalMemory: 6GB (first VM gets 2.5GB, second wants 4GB -> denied) - limits := ResourceLimits{ - MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB - MaxVcpusPerInstance: 4, // per-instance limit (high) - MaxMemoryPerInstance: 8 * 1024 * 1024 * 1024, // 8GB per-instance (high) - MaxTotalVcpus: 2, // aggregate: only 2 total - MaxTotalMemory: 6 * 1024 * 1024 * 1024, // aggregate: only 6GB total (allows first 2.5GB VM) - } - - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) - - // Cleanup any orphaned processes on test end - t.Cleanup(func() { - cleanupTestProcesses(t, mgr) - }) - - // Pull a small image - t.Log("Pulling alpine image...") - alpineImage, err := imageManager.CreateImage(ctx, images.CreateImageRequest{ - Name: "docker.io/library/alpine:latest", - }) - require.NoError(t, err) - - // Wait for image to be ready - t.Log("Waiting for image build...") - for i := 0; i < 120; i++ { - img, err := imageManager.GetImage(ctx, alpineImage.Name) - if err == nil && img.Status == images.StatusReady { - break - } - if err == nil && img.Status == images.StatusFailed { - t.Fatalf("Image build failed: %s", *img.Error) - } - time.Sleep(1 * time.Second) - } - t.Log("Image ready") - - // Ensure system files - t.Log("Ensuring system files...") - err = systemManager.EnsureSystemFiles(ctx) - require.NoError(t, err) - - // Check initial aggregate usage (should be 0) - usage, err := mgr.calculateAggregateUsage(ctx) - require.NoError(t, err) - assert.Equal(t, 0, usage.TotalVcpus, "Initial vCPUs should be 0") - assert.Equal(t, int64(0), usage.TotalMemory, "Initial memory should be 0") - - // Create first VM: 1 vCPU, 2GB + 512MB = 2.5GB memory - t.Log("Creating first instance (1 vCPU, 2.5GB memory)...") - inst1, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ - Name: "small-vm-1", - Image: "docker.io/library/alpine:latest", - Vcpus: 1, - Size: 2 * 1024 * 1024 * 1024, // 2GB (needs extra room for initrd with NVIDIA libs) - HotplugSize: 512 * 1024 * 1024, // 512MB - OverlaySize: 1 * 1024 * 1024 * 1024, - NetworkEnabled: false, - }) - require.NoError(t, err) - require.NotNil(t, inst1) - t.Logf("First instance created: %s", inst1.Id) - - // Verify aggregate usage increased - usage, err = mgr.calculateAggregateUsage(ctx) - require.NoError(t, err) - assert.Equal(t, 1, usage.TotalVcpus, "Should have 1 vCPU in use") - assert.Equal(t, int64(2*1024*1024*1024+512*1024*1024), usage.TotalMemory, "Should have 2.5GB memory in use") - t.Logf("Aggregate usage after first VM: %d vCPUs, %d bytes memory", usage.TotalVcpus, usage.TotalMemory) - - // Try to create second VM: 2 vCPUs (would exceed MaxTotalVcpus=2) - // Note: 2 vCPUs alone is fine (under MaxVcpusPerInstance=4), but 1+2=3 exceeds MaxTotalVcpus=2 - t.Log("Attempting to create second instance (2 vCPUs) - should be denied...") - _, err = mgr.CreateInstance(ctx, CreateInstanceRequest{ - Name: "big-vm-2", - Image: "docker.io/library/alpine:latest", - Vcpus: 2, // This would make total 3, exceeding limit of 2 - Size: 256 * 1024 * 1024, - HotplugSize: 256 * 1024 * 1024, - OverlaySize: 1 * 1024 * 1024 * 1024, - NetworkEnabled: false, - }) - require.Error(t, err, "Should deny creation due to aggregate vCPU limit") - assert.Contains(t, err.Error(), "exceeds aggregate limit") - t.Logf("Second instance correctly denied: %v", err) - - // Verify aggregate usage didn't change (failed creation shouldn't affect it) - usage, err = mgr.calculateAggregateUsage(ctx) - require.NoError(t, err) - assert.Equal(t, 1, usage.TotalVcpus, "vCPUs should still be 1") - - // Clean up first instance - t.Log("Deleting first instance...") - err = mgr.DeleteInstance(ctx, inst1.Id) - require.NoError(t, err) - - // Verify aggregate usage is back to 0 - usage, err = mgr.calculateAggregateUsage(ctx) - require.NoError(t, err) - assert.Equal(t, 0, usage.TotalVcpus, "vCPUs should be 0 after deletion") - t.Log("Test passed: aggregate limits enforced correctly") -} +// Note: Aggregate resource limits are now handled by ResourceValidator in lib/resources. +// Tests for aggregate limits should be in lib/resources/resource_test.go. // cleanupTestProcesses kills any Cloud Hypervisor processes started during test func cleanupTestProcesses(t *testing.T, mgr *manager) { diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 8acc646..ed81fbf 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -51,6 +51,16 @@ func (m *manager) restoreInstance( return nil, fmt.Errorf("no snapshot available for instance %s", id) } + // 2b. Validate aggregate resource limits before allocating resources (if configured) + if m.resourceValidator != nil { + needsGPU := stored.GPUProfile != "" + totalMemory := stored.Size + stored.HotplugSize + if err := m.resourceValidator.ValidateAllocation(ctx, stored.Vcpus, totalMemory, stored.NetworkBandwidthDownload, stored.NetworkBandwidthUpload, stored.DiskIOBps, needsGPU); err != nil { + log.ErrorContext(ctx, "resource validation failed for restore", "instance_id", id, "error", err) + return nil, fmt.Errorf("%w: %v", ErrInsufficientResources, err) + } + } + // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) diff --git a/lib/instances/standby.go b/lib/instances/standby.go index ba19df9..f3b1cfa 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -47,6 +47,12 @@ func (m *manager) standbyInstance( return nil, fmt.Errorf("%w: cannot standby from state %s", ErrInvalidState, inst.State) } + // 2b. Block standby for vGPU instances (driver limitation - NVIDIA vGPU doesn't support snapshots) + if inst.GPUMdevUUID != "" || inst.GPUProfile != "" { + log.ErrorContext(ctx, "standby not supported for vGPU instances", "instance_id", id, "gpu_profile", inst.GPUProfile) + return nil, fmt.Errorf("%w: standby is not supported for instances with vGPU attached (driver limitation)", ErrInvalidState) + } + // 3. Get network allocation BEFORE killing VMM (while we can still query it) // This is needed to delete the TAP device after VMM shuts down var networkAlloc *network.Allocation diff --git a/lib/instances/start.go b/lib/instances/start.go index 9f495f5..9eaf60e 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "go.opentelemetry.io/otel/trace" @@ -45,6 +46,16 @@ func (m *manager) startInstance( return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + // 2b. Validate aggregate resource limits before allocating resources (if configured) + if m.resourceValidator != nil { + needsGPU := stored.GPUProfile != "" + totalMemory := stored.Size + stored.HotplugSize + if err := m.resourceValidator.ValidateAllocation(ctx, stored.Vcpus, totalMemory, stored.NetworkBandwidthDownload, stored.NetworkBandwidthUpload, stored.DiskIOBps, needsGPU); err != nil { + log.ErrorContext(ctx, "resource validation failed for start", "instance_id", id, "error", err) + return nil, fmt.Errorf("%w: %v", ErrInsufficientResources, err) + } + } + // 3. Get image info (needed for buildHypervisorConfig) log.DebugContext(ctx, "getting image info", "instance_id", id, "image", stored.Image) imageInfo, err := m.imageManager.GetImage(ctx, stored.Image) @@ -81,6 +92,26 @@ func (m *manager) startInstance( }) } + // 4b. Recreate vGPU mdev if this instance had a GPU profile + // Note: GPU availability was already validated in step 2b + if stored.GPUProfile != "" { + log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) + mdev, err := devices.CreateMdev(ctx, stored.GPUProfile, id) + if err != nil { + log.ErrorContext(ctx, "failed to create mdev", "instance_id", id, "profile", stored.GPUProfile, "error", err) + return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) + } + stored.GPUMdevUUID = mdev.UUID + log.InfoContext(ctx, "created vGPU mdev", "instance_id", id, "profile", stored.GPUProfile, "uuid", mdev.UUID) + // Add mdev cleanup to stack + cu.Add(func() { + log.DebugContext(ctx, "destroying mdev on cleanup", "instance_id", id, "uuid", mdev.UUID) + if err := devices.DestroyMdev(ctx, mdev.UUID); err != nil { + log.WarnContext(ctx, "failed to destroy mdev on cleanup", "instance_id", id, "uuid", mdev.UUID, "error", err) + } + }) + } + // 5. Regenerate config disk with new network configuration instForConfig := &Instance{StoredMetadata: *stored} log.DebugContext(ctx, "regenerating config disk", "instance_id", id) diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 20888e8..973f764 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "go.opentelemetry.io/otel/trace" @@ -71,10 +72,20 @@ func (m *manager) stopInstance( } } - // 6. Update metadata (clear PID, set StoppedAt) + // 6. Destroy vGPU mdev device if present (frees vGPU slot for other VMs) + if inst.GPUMdevUUID != "" { + log.InfoContext(ctx, "destroying vGPU mdev on stop", "instance_id", id, "uuid", inst.GPUMdevUUID) + if err := devices.DestroyMdev(ctx, inst.GPUMdevUUID); err != nil { + // Log error but continue - mdev cleanup is best-effort + log.WarnContext(ctx, "failed to destroy mdev on stop", "instance_id", id, "uuid", inst.GPUMdevUUID, "error", err) + } + } + + // 7. Update metadata (clear PID, mdev UUID, set StoppedAt) now := time.Now() stored.StoppedAt = &now stored.HypervisorPID = nil + stored.GPUMdevUUID = "" // Clear mdev UUID since we destroyed it meta = &metadata{StoredMetadata: *stored} if err := m.saveMetadata(meta); err != nil { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index a5f6066..935d2d9 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -712,6 +712,9 @@ type ResourceAllocation struct { // DiskBytes Disk allocated in bytes (overlay + volumes) DiskBytes *int64 `json:"disk_bytes,omitempty"` + // DiskIoBps Disk I/O bandwidth limit in bytes/sec + DiskIoBps *int64 `json:"disk_io_bps,omitempty"` + // InstanceId Instance identifier InstanceId *string `json:"instance_id,omitempty"` @@ -758,6 +761,7 @@ type Resources struct { Cpu ResourceStatus `json:"cpu"` Disk ResourceStatus `json:"disk"` DiskBreakdown *DiskBreakdown `json:"disk_breakdown,omitempty"` + DiskIo *ResourceStatus `json:"disk_io,omitempty"` // Gpu GPU resource status. Null if no GPUs available. Gpu *GPUResourceStatus `json:"gpu"` @@ -3732,6 +3736,7 @@ type CreateInstanceResponse struct { JSON201 *Instance JSON400 *Error JSON401 *Error + JSON409 *Error JSON500 *Error } @@ -5426,6 +5431,13 @@ func ParseCreateInstanceResponse(rsp *http.Response) (*CreateInstanceResponse, e } response.JSON401 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON409 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -8703,6 +8715,15 @@ func (response CreateInstance401JSONResponse) VisitCreateInstanceResponse(w http return json.NewEncoder(w).Encode(response) } +type CreateInstance409JSONResponse Error + +func (response CreateInstance409JSONResponse) VisitCreateInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(409) + + return json.NewEncoder(w).Encode(response) +} + type CreateInstance500JSONResponse Error func (response CreateInstance500JSONResponse) VisitCreateInstanceResponse(w http.ResponseWriter) error { @@ -10573,167 +10594,168 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x9a3PburXoX8Hwnk7lU0mWH3Ecnenccewk222c+Maxe0+3chWIhCRskwADgHKUTL72", - "B/Qn7l9yBwsAXwIlOg8nPjudzg4tgngsrDcW1voYhDxJOSNMyWD4MZDhnCQYHo+UwuH8isdZQl6RdxmR", - "Sv+cCp4SoSiBRgnPmBqnWM31XxGRoaCpopwFw+Acqzm6mRNB0AJ6QXLOszhCE4LgOxIF3YC8x0kak2AY", - "bCdMbUdY4aAbqGWqf5JKUDYLPnUDQXDEWbw0w0xxFqtgOMWxJN3asGe6a4Ql0p/04Ju8vwnnMcEs+AQ9", - "vsuoIFEw/LW8jDd5Yz75jYRKD360wDTGk5ickAUNySoYwkwIwtQ4EnRBxCoojs37eIkmPGMRMu1Qh2Vx", - "jOgUMc7IVgUYbEEjqiGhm+ihg6ESGfFAJoI5jWnk2YHjU2Reo9MT1JmT99VBdh9ODoPmLhlOyGqnv2QJ", - "Zj0NXD0t1z+0Lff9fN/XM+VJko1ngmfpas+nL8/OLhG8RCxLJkSUezzczfujTJEZEbrDNKRjHEWCSOlf", - "v3tZnttgMBgM8e5wMOgPfLNcEBZx0QhS89oP0p1BRNZ02Qqktv8VkL64Oj05PULHXKRcYPh2ZaQaYpfB", - "U15XGW2qu+LD/8cZjaNVrJ/on4kYUyYVZg04eGpfanDxKVJzgux36OoMdaZcoIhMstmMstlWG3zXDCsm", - "ikRjrFaHg6ki24ZyhhRNiFQ4SYNuMOUi0R8FEVakp9+0GlAQvGE43aLVYKuklpmdHCeyqXfXBFGGEhrH", - "VJKQs0iWx6BMHew3L6ZEMEQI7uFQT/TPKCFS4hlBHc02Ne9mSCqsMomoRFNMYxK12iMfIpjF/MYniEaE", - "KTqlVfo26NTDk3Bnd8/LOxI8I+OIzqwkqnZ/Ar9rFNP9KASt/QvRhLZstw4YUpDp6nhPgXXDIIJMiSAa", - "x79wuFTwBWGaWvR4/wHjBv9ruxDR21Y+bwMwz4vmn7rBu4xkZJxySc0MVziXfaPRCECN4Av/nOHVur0u", - "YZRUWKynD2jxFSjRzK8VbC5M0zo/BHZnu6lQdiPbe7IgzKP4hJwp+6K64ud8hmLKCLItLHw1n9MD/DXm", - "wOa+xtq6QQHSVYLW8/4MhmR+aOhNv+sGhGWJBmbMZ2VozgkWakIqwGwQS7ajYnaN4D+vkERN/mBJxuu5", - "wjlljERIt7TEalqiTIL2ubJ8oIxrqsYLIqSXjmBaf6cK2RaNXcU8vJ7SmIznWM7NjHEUAQ3i+LyyEo8G", - "VlFpcaoZm+sQNAOJFEcXvxztPjhAdgAPDCXPRGhmsLqS0te6e9MWKSwmOI69uNGMbreXu6sY4seAi5ww", - "muRJjoEOMQ33Cuxu6u67QZrJuXkCfqxnBfJMswGNXrF+fuNZ9DEwCaP5N9pBfr3uZWo2G81irmG6RBmj", - "77KK0txHp1r/V0gzfxqRqIswvNBsGGeK92aEEaH5FJoKnoAGVVJsUYf0Z/0uGmldr6c12x7e7Q0GvcEo", - "qKqm8X5vlmYaFFgpIvQE/9+vuPfhqPfPQe/Rm+Jx3O+9+ct/+BCgrbbtND27zo6j/S5yky2r4PWJrlfP", - "12i4Pi5itu9U0/5td+/4dFXAm/lHPLwmok/5dkwnAovlNptR9n4YY0Wkqq5mfduN64O5rVkYm+ml33Jp", - "NYMD0K0T8xsiQs0pY6IRRHY1s6RKdhHWNiswGaSl2X+hEDONs0awc4EIi9ANVXOEoV0VAsmyh1Pao2aq", - "QTdI8PvnhM3UPBge7K3go0bGjn3ovflP99PW//aipMhi4kHGVzxTlM0QvDbSd04lKuZAFUk2ilsH3SwG", - "FSuh7NR8tpPPBAuBl/5dc5Nbt3vGOGrcPkNAnvWdOLNeImsqgkDA4LSB9T47v9zWJJliKdVc8Gw2L+/K", - "r44fvCnBokEbcIvsBhGV12PKx5PUNycqr9Hp9kukuRWKaUJVwZ12BoOzx9tyFOg/Hrg/tvroxHhzYPp6", - "8VxYpinnWBAQ3RHiDB2fXyIcxzy0xtBUa1hTOssEifo1Gxx692ELYYsvkMNP2IIKzhKtCy2woJp4Kp6F", - "j8GLlydPxk9eXAVDvZNRFloz/fzlq9fBMNgbDAaBT9TpndiAjM/OL49hxbr9nKs0zmZjST+Qik8s2Hv2", - "OKhP/ChfL0pIwoXRR20fqDOvsgMjrlFMrwka6f7Mpu08qzPqXRhqBWjzZUrEgkqfnflL/k7vdyZJmTYN", - "MVRRQhKxICLfa9j8fknWhzHPol5pyG7wjiSA1sVEPY38tl4rKbCBveM4pYw08vfuj8KTb7i4jjmOejtf", - "mSUzonTfq0t8YV5UN9MiAMn3P+iu6PksuqGRmo8jfsP0lD28x75BeeOcAb3XK8Hx7//699VZoYDsPJuk", - "lhvt7D74Qm5U4z+6a69xkS8kS/3LuEz9i7g6+/1f/3Yr+b6LIEzjZ1RhOsZery7lH3Oi5kSUpJLbYP2T", - "0Q7hc+TwpTR8xQFQ9tqvME6+ICLGSw8j3Bl4OOE/BFVAX/Y7pCUa0h9vYIO6Nye8VhnhwM8JPZPyzOmx", - "pm/Ll9vMJJ/Izu6Zfdxty5vlNU3HM61sjPEsd2CsO0+5uKYpgi968IXZxjg2xBtlumc04Vz1R+wfc8IQ", - "7B1sMHlPQuBT2kJDR+enEt3QOAZzBxjBKu8fsdclVmCaS6X/KzLWRZNMIUESroi2NRPdtx4kg7lA4wlB", - "GcPuwKY/YmWo2AXW8cqC5ZoIRuLxnOCICNkSMuYjZD9qBA4sdYqlIsJw6Cytwuvk72cXqHOyZDihIfq7", - "6fWMR1lM0EWWahreqkKvO2KpIAvCQNHVCgO14/Ip4pnq8WlPCULcFBPoLDcY7WnC4tn5pT2Pklv9EXtF", - "NGAJi7S9yQVyUkIiNccKRZz9WVMsiardlsevAd1Py91gEaZZFcq7dQi/gFMgvZ4FFSrDsWZZFY3Leyhk", - "jhs9Gqo5zSxrypYV5QiHVdWb39ZSMD3D2eOq3uw3DozC0WwcbDh69fnYc4dDmEnFk5KnHXVqvgRa9TpU", - "mceCx70IKwyqQUv9xUx39dQqWZquzKY0ccnxbOJxUGlmSBma0RmeLFVV194ZrG69H9Cufx+om050DXqQ", - "aKz4+jMtOkWubRsXNpz/jhUfL6bU03MuNAvnCZUorB0fW6TVXfTSkFry7aKbOdViViIHBKDgq7OyDdgf", - "sR6wnCE6yQfIu8271JwVHGXQRYeL0iQo+DzRZLmFMLo666PX+Wz/LBHDii6IO+KeY4kmhDCUgXpGIhgf", - "2Gl5ApnUPIyq+ueWV5nT8C0wdbl910falkiw5fsavROsaAh+tgmtrQfON8xG6ZE0A2BlqdNKSqw7CXxF", - "ZlQqUTsHRJ1XT4/39vYe1fWF3Qe9wU5v58HrncFwoP//z/ZHhl//wN/X11GVX1jPZZmjHF+enuxa5aQ6", - "jvqwjx8dvn+P1aMDeiMffUgmYvbbHr6TkAA/ezopXK6ok0kieo71aazyOVpL/swGR+pn+0dvFY3gTmTW", - "iR+zute65beIX/CdotkznNtHGNSZ4MZzuNLiVtajf9X6QYH5Jd+AdXeH1OvYP6Hy+rEg+FpblR75qsWz", - "HBu54/d1ZdqOmiwRea/VMxIhwbmaSuMvqKopO/sP9w/3DvYPBwPPsf0qEvOQjkMtVVpN4OXxKYrxkggE", - "36AOGHoRmsR8UkXeB3sHhw8Hj3Z2287DmEnt4JBrUe4r1LEQ+YsLAXNvKpPa3X14sLe3Nzg42N1vNSur", - "4LWalFMGK6rDw72H+zuHu/utoOAzO5+4MIr6sXDkQdKjNI2pMbJ7MiUhndIQQSAG0h+gTgJiieQWX5Um", - "JzgaC6sGeuWBwjT2gKHk9TOD2ZYm6ibJYkXTmJh3sCGtNF1Y+Qn05PMQU8aIGOdRJrfoyQafbPSMubXk", - "TVAliKgCujMqQbMoFCJK4mhoKHQjn4PdLCb2pgkP7BpaYsNzfkNELyYLEpeRwIgjPdmEC4JyPDGbVlkV", - "ZQsc02hMWZp5UaIRlE8zAfql6RThCc+UMdVhw8qDwJEZ2AhTza7bndgWPuqVobWdeUvHXyr4lMaeZYDR", - "at9ake5cYs/3Bxe9nf8DfrCXLF4aPkCZMXQTHpF+LU4R2rde3nnTnPIgUVSe3cqacteExz2aW7sOItbo", - "DjFDE4KsmDROXXCbFIMUDP6Rj2FOBU7IJJtOiRgnHkvrqX6PTAPjg6IMnT2uMk3NnNuqW+eVzQF9a4pD", - "G+PXDvoeS662jG4Jmm/82/WKmLCGpigCvVXCtrGBBH30Ig/LRc/OLyUq3EkeE6/lgd35fCm1cWJ6NEFB", - "lJUtM0DO1mz4vPjQ2rAeZpx4GZAjBNRZzNIMyPDiVe/05dV2EpFFtzIncAHNeUz0vLdKutXCxRIUp4uV", - "I5dFk4psEEO2JaASrHIKbg2kEr16oKO4wvFYxlx5ZvNav0TwEnWunpozZD2DLkorW6l/L0Ghgt8HXorR", - "HKlp2AsYsG5rVwh8o9sjMWKrvLzKoD5S+YXg2ATxV/G5CEtzG8+vqxvNrzdSr+3EN+6pO3WrSc7EY7sc", - "n50YyyzkTGHKiEAJUdheGSidbEOARdANeloZiDBJwCc6/a/1Z90NvpscXdZZ/8crEcDfxPJviHLTTC5e", - "kAglmNEpkcpGuVVGlnO8++BgaOJrIzLdf3DQ7/f9JzxKLFNOfeGNT/J37bZi25yP9oo++3L+ZfvwDc7w", - "26zlY3B+9PqXYBhsZ1JsxzzE8bacUDYs/Z3/WbyAB/PnhDLv2X+rkGw6XQnFrmxvqmWW+X2oV8JImCMk", - "By1xo2/SL8lfaNSM6QcSIW9ElMIzpPVvwLgvC336giDm4iaNKgUvl48JWgQy0w/rzW2nGEEbO2bGFI2L", - "GO9VQ/uzovTl2qDHlYDHlLA8zDGOzVPI2UJThS/mscLA3buVzbjh4pqy2TiiHuz8h3mJIipIqCCkZDMN", - "Bds4TTejol/5y3la2/htG73lkS7fnZN/jsO1OvrL2d/e/V95/vC3nXfPr67+e/Hsbycv6H9fxecvvyjk", - "ZH3g3neNvlt7pgZexkrUXVv0OMMq9Cg+cy5VA9TsG6Q4SvTHfXQMBtpwxHroOVVE4HiIRgFOad8Csx/y", - "ZBSgDnmPQ2W+Qpwh3ZU9Ot7SH5+bsBv98UdnA36q9xHZM2JhgZyHc8hsEvEEU7Y1YiNm+0JuIRIObfRT", - "hEKcqkwQvSNa14yXaCJwWJwNF4N30Uecpp+2RgwsUfJeCb2CFAuVR/m6EWCj7azMoZBtTiK0wHFGpLVk", - "RyyXH2Ca604UFjOi+rkLERw1tYOZBqB4zQwuqrENh4OuZx+Rbqc3MqZSEYZyrwSVgLyo44JUDgcV8j8c", - "HG4+f8xxaA36AXav3qt1SNmCPgwCw9CGGY/nSqWbwxeA3xgaQb+8fn2uwaD/vUCuowIW+RYbYwynaUyJ", - "NKdqKgadxMYFbQW+kzOzuy0X9No01p/FLcIwnsDA6PXzC6SISCgz/LsTanBOaajXB+c7VMpMoyLF6Oj4", - "7MlWv8XFYIBtPv81+/g6X2HtGME5t1YtTPiicJpr+HbR6UlXq1OWQgtFC85Nn3KBYsNgCroeoktJqlEM", - "sFXmiMfsZLwsPGSGq4+CLddjWucUQ/Qq1+9wPpX89kGBDK7Lgi6hWxvYYg51V3rvVucKx9XWfrGsDY5w", - "sULW6Q2iuJkVrCd/D8SB5jmr+x5vR9tlp6UezI8axd5/cw1k77a25G0juatBaaUgxDyY+/tGYX9OTLXb", - "oWfnlxC5jOVYMpzKOVfNwRkYuTaIvKdSydU4tlbhBKsx3FXxZKKz1wQGfs1obJExBpER9WV89Tjr7xlr", - "8OPFeK+Nyv7S0GqroH2jyOpGhuCLSq7yBvPz142R/ibTqUQ7+5hBWY65QLDPDnDuBtQTBHMkJZ0xEqHT", - "8+LWX+HwcN3X1vRot79zcNjfGQz6O4M27p8Eh2vGPjs6bj/4YNcYxEM8GYbRkEy/wP1kEdsoHDi+wUuJ", - "Rk4lHAVGBy0pnyWytWpjq6O91TjyzwsbrwvBTYHhtwkEbxfhveY6/kX1In5rveLBP7/ozj5pK4YvoLH7", - "anwbxyhBIc/iiP1ZoYmmPGMKkMhaLJKoIscBEOslu2b8hlWXbvxjmn7fZUQs0dXZWcWbKsjUXvdusXCe", - "po37wNNbbcPuBvVu42xKwdZ3EWBd54QlCfTVw6nLrh8X12GwroULqFD/vMeklBlw671fs6aa8R6RxTjL", - "fIqOfuUiNC8vT08qG47xwc7h4PBR73Cyc9DbjwY7Pbyzd9DbfYAH073w4V5DYpT2YRKfH/lQpdDmiGgA", - "PDjCTBB7NNQ0lIcuTDKF8ktqmjiPtcaISnqoif8F2/SVUUl1DyBdQ/0mXuaq6tqPz7EmVPdtCn+t/+Ji", - "nimtBsE3cp4ppP+CKeslWFV/fReG5ofoBYdv7Ey7WlDWbAbTHLNoslxtXrcvOjYCRBCpuCARDGYZ2BA9", - "zZlWzvYsm+tIYh8NL7WRUhAFtmUMaqve290KuoGFetANDAiDbuAgox/NCuEJJh90AzsRb5BlGW98jmKC", - "Y+BhRRBGpmhMPxiS01OnUtHQmFgYdrOJ7OxNNhKNjQhtOsoxJ/tWzOYfOaq+OkMduHfwF2QtMP3XVn7s", - "Uyah/d1H+48OHu4+OmgVtVhMcDM3Poa4k9XJbWTNYZqNXYKohqUfn1+C8NGCTWaJCZO0ay9sN804Qq3t", - "UYaKjFPF4I/6j8rBmhHPJnHJ02CjtSEisE16sIZzjnc0XtDplL37EF7v/iZosvP+QO5OvMZRPpBfkzwt", - "e8dWzC4y6Zmrxv54OkAoIRtDTl8RCStAF0QhwJ+eZlhaoubhIhblXGCqhbgXsfb39vYOHz7YbYVXdnYl", - "whmD/bc6yzM7gxKJQUvUeXVxgbZLCGf6dDF0qSBSL87covDSGRplg8EeQYNKeJ22PfZ8WNKgsBRYY/te", - "JI0gv7Iai12UBTpEveTazAqVe6G9tzd4uP/g8EE7MrYWz1i8X89hbDt7WixISOiisvMd8Ki+PjpHuncx", - "xWFVw9/Z3dt/cPDw8FazUrealRKYyYQqdauJHT48eLC/t7vTLnba5zW1twIqBFvlXR6i8yCFZzc8oFhl", - "vd0maeHTEldD7dZG9xXhgvXYsNsEgxY3waiEXmkpDhF1tBJVVkhLt5m22vgZ/CxSj9OUdlKri23jNNeH", - "ZZ5jNT9lU77qFr+NwWeDXdwhRKoVHwkJuSLCKIkc78otP6tLQfhMLAmKMmIhZ3QjgS3AsTkaSLGag7IK", - "H1I2qwYOrwzYxgwzc1h/7w/GtQ3beIykP0DjtcgAVsahKxEuQjVaeaepHPutitWOBZllMRaoHou8Zspy", - "mcSUXbfpXS6TCY9piPQHdXN+yuOY34z1K/lXWMtWq9XpD8bFqWTNPDeTs2fSZkNq4xZL+Kte5VYtygUk", - "/7b5fhvyCrdxwHmjdZ9q482E614y+r6E6NVLNPu7g6agpoZOK+FMq6Het+XtFmV9FO+isI/yZBOeIzFz", - "WlOzYKt6cGW9vtXCqda6EK5VTQB1nE/PXVKqwrV0WaiVIP4idXdN9tMvUG4XyRqltgFaZ200p8MHh48e", - "7e0/eLR7Kx3FHTY0HD42HTi4GWxLEtbyutT0pgcD+N+tJmWOG/xTajhyqE6okqPlsyf0aQ35FJcYGizf", - "dZnHi510pnZVCW6nZq7RWI4qak8pVVeHTKcEnDdjA7deMZlaUE2rOYQ4xSFVS48Vhm8gzgDlTWrB+C16", - "r03WA1LbN8JTpa31BREymxS30TpucPSfxrqq4cJh64uXMps0WXIv66MaO84E5kQ1L0ELI91ghO8U+iYH", - "JrrBsuJZ18+hIlG3lIqtfgRjWrTPNOtwPU82Wxwu+y6U+BPLlre/tp0lzb+iqNYhvk6MNZOglsoQ9dPG", - "ye2Rip5bKuHmKIYaf7By8PO+Gk/KV6LX3jmv3J9uncNudVgjiG4/3dJh+G0+rF/yBLSyc7CQK/ruVnbW", - "hxTmqKIpg0jiKmvU7oBSk6vcXvRBpcaoQ5JULV0wuzPytm53dHKUd+jFqa8cfjR49DUCoC/XRjz/D8lJ", - "Uz6tcoNsPKda2dPGMEO/1nlSjwQx5pW9k1+NXKjdNJZqTUr+deVfTB0WsJ1siO8sq99JukXJlyZruaAc", - "l2vf1XzZZASudU2VVlaaSfPemKPKL6yPQ6UrjPOZILOWzOaYWXPco23JXj1pg7nyKSiYRhZABrAaBLm1", - "u2pSr4+gOMPv8xHA8MQS1dLcmXWUUsY+ewzXuF+5y/t06rqAadQTFj7+ssJBDqtWN2NdJSF3GO4lPMt/", - "1nC0JtqqIWcxRnd9sSLNukiYCaqWF1og2DgvggURR5lBQ5AUsAj4uRgc4sY/fQJrc+pROp8RRgQN0dH5", - "KWBJghme6S27OkMxnZJwGcbEhv2uHJPCrfWXx6c9c18hzysHdQAUAMQldDo6P4VcMjYDfzDo7/YhbS5P", - "CcMpDYbBXn8HsuVoMMASt+E6GDxan46mQ5Bkp5GVuI9NEw1amXImDXB2B4NaRQdc5OvY/k0aZ4URr611", - "O1MyZzV0YSWa1WkCdvqfusH+YOdW89mYYsM37CXDmZpzQT8QmOaDWwLhswY9ZcY4dkl9iW1Y4Gww/LWK", - "rb+++fSmG8gsSbBWEQ24ClilXDapMEQijBi5sfcEf+OTProwpgXk2yiKkRnLn0SaJWGksOjPPiAswjld", - "kBGznNikS8ECLkUkSHNgE5JeRTMztNl9Q8JEqsc8Wtagm3e3rbsDbaQK4FuXushz/6UNNS983NGkGJIh", - "9+ZWIgwzVWSsMbmFrgmcB07pe29YOYTK+h3HJ/k7Vxylytu1uktZGGdRIQCrRSm815VNcQWbLemaePSF", - "Z9DCzr8cVewkDeMRMRGi6VLNOTPP2SRjKjPPE8FvJBFaHtnbDRYs2vrNi1qZVHQ0gRsG5j6kHnPbTHH7", - "4zVZfuqP2FGUuPurNiMqjiW3aaTMWT+VKM/LO2JeDVqOse5nPHHVuWqKKoGuRoEWlaNAP88E1ipZJucI", - "h3C2r38sA6djsJkLEHdb9bmGmKGUp1mslQfYHpNnqtIHXBTDcYwU4I/7VgtRgEnDeiQJBfHZSn+7ePkC", - "Af+EKiXQrAjUhjVQpqVfnm9VD9gfsSc4nCMjGCEP4Sig0SgoqlFsgRDLJDGyqdcDyfpXKNNjhunS6K/9", - "vu7KCO0h+vWj6WWosSZNxopfEzYKPnVR6cWMqnk2yd+9aVhwg8vlooLyqGMY0pa7WqtXWOLNhplhFiFu", - "GUC8RBgVtFY2ySaUYbFsKu3CM9UcOmJuHttmxbW4g8Fga/PRgF2qR12pNNSY+mlFOu9+NcFkhfKqYCqV", - "cdNigNlr5ZERx3cgGR/jyN12+qkCbFABrO1SEu7wvVUAtz/S6JNB35iYUMWahIZqP05Cp1jghCjI9/yr", - "H+chSpPqv91BHvgajCVfRd5uCTx1hf7NCmLvN5ZRygsSAS7s3wH+wbhFsi8Y99FdjYtjk2o2L+14r9AR", - "NsshYtdvfTwj6kfAuMFdsVKXk/A74u99wZ9nxKpIBdBq3GwbkryXTdv6bQJBcCJtL6axtmUuYE69C8IU", - "ggJ+sm//dWo2BGq/jfns7RAZEMa2fKG0WeZyH7AWihaW8JFJwpF/Z3PThHPMZkSijpGfv//r364E2+//", - "+rctwfb7v/4N5L5tC4pCd3nxwLdD9HdC0h6O6YK4xUDwIVkQsUR7A1vWAl55Mt3IERuxV0Rlgsk8dEev", - "C2BiOgSVncF6KMuIRBJACDmopzamxLiYPCaeo2UDyjul6O6KpWtXUFqAlooOB+CAkjKqKI4Rz5RJFwnz", - "gPstxUTMmoPy4HVv2Yr/dDN/UeS9MtjbMxO8JYMxxTc9dGfqUZo+Uefi4slWH4G6b7AC4obAbii6sZZA", - "/ydP2syTDEepMhSAsuFNpSSHjb62E9vmLpxtTQkQm71tArK1E227usX8VLtbeN78cHNeOJ8r7MQl5W72", - "hX3+en21OVvZlF9vnx3urcLcZpwvQPY9rEnUscmC85wglbT23wvp74QBl6oh5FwYcZOJ5M4snGPOpjEN", - "Feq5udh6jbnVU0WQ+8IOXtlZI+zWVQ92L4uK7UrcWKPQyEPI7lJ61Aa9jRgpAvILXPspSTahzgmVIdff", - "lrClF+IUAGmBWNBpGYs2+XZO4Pdc5KxVzPMKqo4g787LY4fOWF023AFTPKkxxO/ICGsZM0pXWO4TNl/m", - "u+gKkKxxAv1YqDm4Oy3orh1CPjS/Tx6hqAY2zQXneY7uJvSyWby/4UbbETwLvyDCUbWZqMnUUCzLfIrC", - "OQmvzYJsnZx1GsGpK6Xz7fUAk4r8FtLfTv+nuG9hOBawWmcsntr0Hd/OVoQRbmUqfr3jR4tgHiBDlMbE", - "OVJNZgwslyzc+kOdQN6JZKjXtblHlHSexbFzxC+IUEVC9jI/3f6o9YMWerKjtrW6yOWr5z3CQg4xOQZ0", - "jQqJy7/8dbVls2FmKT/RpI19BaByiNGsjH7B/pvQKZQnRvzT7lObGvFPu09NcsQ/7R2Z9Ihb3wxZBnfF", - "mu9ae73HyKeVV1oFGrAmkyV5k7aXt7oThc+mo7+NypdP8KfW10brK4NrreKXVwb4hqqfTbj+fc4JcmTz", - "QRteufizP5jKd7euJ4uRpRp6FV+8zRHCRZHk3Fbgun8BcjTHuDL/belDLQhyrXbgUPf0pGvz15us83mA", - "+B15VN087lxLtOPevTv1KJnQWcYzWQ5oh3IFRBa1XSsM+L7pr4V4btRgf2AsHdyl6LhzBfUn3n8j1bm+", - "oYZ523KwG5Rn1+pulOfiqKa99uxm+FN7bqU9l8C1XnvOU6J+S/XZDPLd9GeHbz6A2yvMf0QN+r5d22DW", - "x1067K3wuNYKapEGeL3st7jxPQ7688HvXi916cDuZ/gpNwHnkdMEC1nTrAr+aPgwuFved/cq4H1GsWfl", - "ymt+ZcvcvYj5bPPNi7wnd83Ac/VixFyZtrfmOuRblCMqUhxJEpNQoZs5DedwDUP/Bv2bWxo4Td/m9y63", - "hugZhHeWb4LC4B1JBMUx5PbmsUlL/3aRJG+HqykZrs7O4CNzA8MkX3g7RC4NQ05jUrcqX6vQq4ixVOiF", - "vSzS0RsueBybPMJvNTxL69uyFy6KK6oj5rt8wciN7ZBO0dvSPYy3DRcxHBI+17v0nSi/25zG3axFcSQA", - "cOZKOGFRwyUMDTX/FYydgTezUMvrIGYa3/g2yMpknvNZfn2/gso4Tduir50mYPEiSdbgMOqUUtdLFfFM", - "/UWqiAhTWdVidxNyow4OzR8KX5s6oJVCaKZYgg9U9mqzF1SBqXbsaiyYvxZJEpiqbAn21Uz48ms19Q5X", - "7TG9M6W7Mz9lxm1uxVSZfelaTE1y2GIdkMzDa7y9Mg3+8JqLq2ryndHw7o8iSrOgUOyERZMl7G1RLuZ+", - "3QmAjSxWBvLOrstLI+5dI43YKjN/eBop8OMPTiUhF1CaWrpScfcneKtkcZTIvQO1qYqaT11n9V6dnW01", - "EY2pbtxIMuKnOWzjKP/wMgXKdd0/ajGVKnG+gHXOQk0QqtFGdzZrpZTfhGe695XspFDCQi6lIokx2KdZ", - "DBfbIGrd5gfA5RIdXUSVhGTVXXBZlcozjNiETLU8TInQY+vPIftZYXv4zNoLhXPyPTc0+GPYtZCwFEw5", - "rJqgVquDkaYuV6nPdsrTq372lJ6CoVotESJRJ6bXpu4dWkgU64ettZauqR/ytbMffD5l5RVyfLdaDc7m", - "yPxH4HCnNbbmKkDeO7b2jJSJxfEf2Gg/W5Mb+Zq4ZQlFB7tSKcX+iJ0RJXQbLAgKeRxD1n6jv2+ngofb", - "UN4tTGlk6rzB5IDhNb9OYMTj80toZzKsd0dM/7FaYKw+UVen7HT75Qbfnykt+T9YzzELXEcW/g3/6da5", - "/VFAIw3JBhLl6TpNnKc/FXFbMfan2XovzVY4i81X05kJHIJSLG1NYL+JagtpbX80D6ebTvQVDudXrhjD", - "j6Ht2tztm4ZxC7wXRGnXFBFz6/7uaZLn6fXv6c0qDTi3BFBiyrEJfilgynb80bD764ehleF4qyC0O6Ut", - "l9Hih6Gtu5Z8dg7uRkUZHveFzA2muZVAfvGy90mUy4Cttc1ceSeoSZerlq46WbdcJM8k0Mx9SEVZlrwe", - "V3/E8gJkLoGntq66zrRCEZXXpgdrPfWRv06csfNssbgRUxyFOA5NWve8YJopcigbrK9XpSKC34zeikE8", - "G51XipN5RbD7ZHL4cQJ2r1xyDDDOqlNrw7+vbJu7CP62wuwWod9uBT+jZFsEfpeA1abAiakXZ7mVLfSV", - "V6eAYkv9hjoluVLy7cLGP0Nefz30cHjaKK1/Xrm8M4WguHF5elK7b3kPw9jLNFfh0dvaKujZ6kFl19A6", - "CrYgSgXpufIqkQGYhYexNerFifoj9npO3F+IulBKEtla7/ESUQb1ZFyNuT9LJDhXRS345iJGhkSeCp4c", - "2dVsMF5aV1v0HcTcOh1E11NhjiZZkpc1f/YYikQLE9mHppjGEFfqQEreh4REEnByq17F0Rvql5dr3DjL", - "NTGaeZ2mMJOKJ27vT09QB2eK92aE6b0oSiKlgi9oVK+sWymH6ZstWIhfwUibfaBplfQ2lpNZJbwq3qK8", - "BpStZ1Pgp9ud4KeYqCfw1butjTwHRMU5irGYka2fouQ+i5KyN8nJjYpEaXchqp2DqaXf51tchsqdj3d7", - "Ferqx/GJlBKe3sP7+Ivc6Gu6g/VjoeDg7uTDXd+9urrHPvRnxBm4pXtX0IHu0Ycwz3mIYxSRBYl5CpWe", - "TdugG2QitnVrh9vbsW4351INDweHg+DTm0//PwAA//+QJ/en3+AAAA==", + "H4sIAAAAAAAC/+x9a3PburXoX8Hwnk7lU0mWH3Ecn+nccewk222c+Maxe0+3chWIhCRskwADgHKUTL72", + "B/Qn7l9yBwsAXwIlKg8nPjudzg4tgngsrDcW1voYhDxJOSNMyeDoYyDDGUkwPB4rhcPZNY+zhLwi7zIi", + "lf45FTwlQlECjRKeMTVKsZrpvyIiQ0FTRTkLjoILrGbodkYEQXPoBckZz+IIjQmC70gUdAPyHidpTIKj", + "YDthajvCCgfdQC1S/ZNUgrJp8KkbCIIjzuKFGWaCs1gFRxMcS9KtDXuuu0ZYIv1JD77J+xtzHhPMgk/Q", + "47uMChIFR7+Wl/Emb8zHv5FQ6cGP55jGeByTUzKnIVkGQ5gJQZgaRYLOiVgGxYl5Hy/QmGcsQqYd6rAs", + "jhGdIMYZ2aoAg81pRDUkdBM9dHCkREY8kIlgTiMaeXbg5AyZ1+jsFHVm5H11kN2H48OguUuGE7Lc6S9Z", + "gllPA1dPy/UPbct9P9/39Ux5kmSjqeBZutzz2cvz8ysELxHLkjER5R4Pd/P+KFNkSoTuMA3pCEeRIFL6", + "1+9eluc2GAwGR3j3aDDoD3yznBMWcdEIUvPaD9KdQURWdNkKpLb/JZC+uD47PTtGJ1ykXGD4dmmkGmKX", + "wVNeVxltqrviw//HGY2jZawf65+JGFEmFWYNOHhmX2pw8QlSM4Lsd+j6HHUmXKCIjLPplLLpVht81wwr", + "JopEI6yWh4OpItuGcoYUTYhUOEmDbjDhItEfBRFWpKfftBpQELxmON2i1WDLpJaZnRwlsql31wRRhhIa", + "x1SSkLNIlsegTB3sNy+mRDBECO7hUE/0zyghUuIpQR3NNjXvZkgqrDKJqEQTTGMStdojHyKYxfzGx4hG", + "hCk6oVX6NujUw+NwZ3fPyzsSPCWjiE6tJKp2fwq/axTT/SgErf0L0YS2aLcOGFKQyfJ4T4F1wyCCTIgg", + "Gse/cLhU8Dlhmlr0eP8B4wb/a7sQ0dtWPm8DMC+K5p+6wbuMZGSUcknNDJc4l32j0QhAjeAL/5zh1aq9", + "LmGUVFispg9o8RUo0cyvFWwuTdM6PwR2Z7upUHYj23syJ8yj+IScKfuiuuLnfIpiygiyLSx8NZ/TA/w1", + "5sDmvsbaukEB0mWC1vP+DIZkfmjoTb/rBoRliQZmzKdlaM4IFmpMKsBsEEu2o2J2jeC/qJBETf5gSUar", + "ucIFZYxESLe0xGpaokyC9rm0fKCMG6pGcyKkl45gWn+nCtkWjV3FPLyZ0JiMZljOzIxxFAEN4viishKP", + "BlZRaXGqGZvrEDQDiRRHl78c7z44QHYADwwlz0RoZrC8ktLXunvTFiksxjiOvbjRjG6by91lDPFjwGVO", + "GE3yJMdAh5iGewV2N3X33SDN5Mw8AT/WswJ5ptmARq9YP7/xLPoEmITR/BvtIL9e9zI1m42mMdcwXaCM", + "0XdZRWnuozOt/yukmT+NSNRFGF5oNowzxXtTwojQfApNBE9AgyoptqhD+tN+Fw21rtfTmm0P7/YGg95g", + "GFRV03i/N00zDQqsFBF6gv/vV9z7cNz756D36E3xOOr33vzlP3wI0FbbdpqeXWfH0X4XucmWVfD6RFer", + "5ys0XB8XMdt3pml/0907OVsW8Gb+EQ9viOhTvh3TscBisc2mlL0/irEiUlVXs7rt2vXB3FYsjE310jdc", + "Ws3gAHTrxPyWiFBzyphoBJFdzSypkl2Etc0KTAZpafZfKMRM46wR7FwgwiJ0S9UMYWhXhUCy6OGU9qiZ", + "atANEvz+OWFTNQuODvaW8FEjY8c+9N78p/tp6397UVJkMfEg4yueKcqmCF4b6TujEhVzoIoka8Wtg24W", + "g4qVUHZmPtvJZ4KFwAv/rrnJrdo9Yxw1bp8hIM/6Tp1ZL5E1FUEgYHDawHqfXVxta5JMsZRqJng2nZV3", + "5VfHD96UYNGgDbhFdoOIypsR5aNx6psTlTfobPsl0twKxTShquBOO4PB+eNtOQz0Hw/cH1t9dGq8OTB9", + "vXguLNOUMywIiO4IcYZOLq4QjmMeWmNoojWsCZ1mgkT9mg0OvfuwhbD5F8jhJ2xOBWeJ1oXmWFBNPBXP", + "wsfgxcvTJ6MnL66DI72TURZaM/3i5avXwVGwNxgMAp+o0zuxBhmfXVydwIp1+xlXaZxNR5J+IBWfWLD3", + "7HFQn/hxvl6UkIQLo4/aPlBnVmUHRlyjmN4QNNT9mU3beVZn1Lsw1BLQZouUiDmVPjvzl/yd3u9MkjJt", + "GmKoooQkYk5Evtew+f2SrA9jnkW90pDd4B1JAK2LiXoa+W29VlJgDXvHcUoZaeTv3R+FJ99ycRNzHPV2", + "vjJLZkTpvpeX+MK8qG6mRQCS73/QXdLzWXRLIzUbRfyW6Sl7eI99g/LGOQN6r1eC49//9e/r80IB2Xk2", + "Ti032tl98IXcqMZ/dNde4yJfSJb6l3GV+hdxff77v/7tVvJ9F0GYxs+ownSMvV5dyj9mRM2IKEklt8H6", + "J6MdwufI4Utp+IoDoOy1X2KcfE5EjBceRrgz8HDCfwiqgL7sd0hLNKQ/XsMGdW9OeC0zwoGfE3om5ZnT", + "Y03fli+3mUk+kZ3dc/u425Y3yxuajqZa2Rjhae7AWHWecnlDUwRf9OALs41xbIg3ynTPaMy56g/ZP2aE", + "Idg72GDynoTAp7SFho4vziS6pXEM5g4wgmXeP2SvS6zANJdK/1dkrIvGmUKCJFwRbWsmum89SAZzgcZj", + "gjKG3YFNf8jKULELrOOVBcsNEYzEoxnBERGyJWTMR8h+1AgcWOoES0WE4dBZWoXX6d/PL1HndMFwQkP0", + "d9PrOY+ymKDLLNU0vFWFXnfIUkHmhIGiqxUGasflE8Qz1eOTnhKEuCkm0FluMNrThPmziyt7HiW3+kP2", + "imjAEhZpe5ML5KSERGqGFYo4+7OmWBJVuy2PXwO6n5a7wTxMsyqUd+sQfgGnQHo9cypUhmPNsioal/dQ", + "yBw3ejRUc5pZ1pQtK8oRDquqN7+tpWB6hrPHZb3ZbxwYhaPZOFhz9OrzsecOhzCTiiclTzvq1HwJtOp1", + "qDKPOY97EVYYVIOW+ouZ7vKpVbIwXZlNaeKSo+nY46DSzJAyNKVTPF6oqq69M1jeej+gXf8+UDed6Br0", + "INFI8dVnWnSCXNs2Lmw4/x0pPppPqKfnXGgWzhMqUVg7PrZIq7vopSG15NtFtzOqxaxEDghAwdfnZRuw", + "P2Q9YDlH6DQfIO8271JzVnCUQRcdLkqToODzROPFFsLo+ryPXuez/bNEDCs6J+6Ie4YlGhPCUAbqGYlg", + "fGCn5QlkUvMwquqfW15lTsO3wNTl9l0faVsiwZbva/ROsKIh+NnGtLYeON8wG6VH0gyAlaVOKymx6iTw", + "FZlSqUTtHBB1Xj092dvbe1TXF3Yf9AY7vZ0Hr3cGRwP9/3+2PzL8+gf+vr6Oq/zCei7LHOXk6ux01yon", + "1XHUh3386PD9e6weHdBb+ehDMhbT3/bwnYQE+NnTaeFyRZ1MEtFzrE9jlc/RWvJnNjhSP9s/ulE0gjuR", + "WSV+zOpe65bfIn7Bd4pmz3A2jzCoM8G153ClxS2tR/+q9YMC80u+AevuDqnXsX9K5c1jQfCNtio98lWL", + "Zzkycsfv68q0HTVeIPJeq2ckQoJzNZHGX1BVU3b2H+4f7h3sHw4GnmP7ZSTmIR2FWqq0msDLkzMU4wUR", + "CL5BHTD0IjSO+biKvA/2Dg4fDh7t7LadhzGT2sEh16LcV6hjIfIXFwLm3lQmtbv78GBvb29wcLC732pW", + "VsFrNSmnDFZUh4d7D/d3Dnf3W0HBZ3Y+cWEU9WPhyIOkx2kaU2Nk92RKQjqhIYJADKQ/QJ0ExBLJLb4q", + "TY5xNBJWDfTKA4Vp7AFDyetnBrMtTdRNksWKpjEx72BDWmm6sPJT6MnnIaaMETHKo0w26MkGn6z1jLm1", + "5E1QJYioArpzKkGzKBQiSuLoyFDoWj4Hu1lM7E0THtg1tMSG5/yWiF5M5iQuI4ERR3qyCRcE5XhiNq2y", + "KsrmOKbRiLI086JEIyifZgL0S9MpwmOeKWOqw4aVB4EjM7ARJppdtzuxLXzUS0NrO3NDx18q+ITGnmWA", + "0WrfWpHuXGLP9weXvZ3/A36wlyxeGD5AmTF0Ex6Rfi1OEdq3Xt5F05zyIFFUnt3SmnLXhMc9mlu7DiLW", + "6A4xQ2OCrJg0Tl1wmxSDFAz+kY9hTgROyDibTIgYJR5L66l+j0wD44OiDJ0/rjJNzZzbqlsXlc0BfWuC", + "Qxvj1w76HkuutoxuCZpv/Nv1ipiwhqYoAr1VwraxgQR99CIPy0XPLq4kKtxJHhOv5YHdxWwhtXFiejRB", + "QZSVLTNAztZs+KL40NqwHmaceBmQIwTUmU/TDMjw8lXv7OX1dhKRebcyJ3ABzXhM9Ly3SrrV3MUSFKeL", + "lSOXeZOKbBBDtiWgEqxyCm4NpBK9eqCjuMLxSMZceWbzWr9E8BJ1rp+aM2Q9gy5KK1upfy9BoYLfB16K", + "0RypadhLGLBua1cIfK3bIzFiq7y8yqA+UvmF4NgE8VfxuQhLcxvPb6obzW/WUq/txDfumTt1q0nOxGO7", + "nJyfGsss5ExhyohACVHYXhkonWxDgEXQDXpaGYgwScAnOvmv1WfdDb6bHF1WWf8nSxHA38Tyb4hy00wu", + "npMIJZjRCZHKRrlVRpYzvPvg4MjE10Zksv/goN/v+094lFiknPrCG5/k79ptxbY5H+0Vffbl7Mv24Ruc", + "4bdZy8fg4vj1L8FRsJ1JsR3zEMfbckzZUenv/M/iBTyYP8eUec/+W4Vk08lSKHZle1Mts8zvR3oljIQ5", + "QnLQEtf6Jv2S/IVGzZh+IBHyRkQpPEVa/waM+7LQpy8IYi5u0qhS8HL5mKBFIDP9sNrcdooRtLFjZkzR", + "uIjxXja0PytKX64MelwKeEwJy8Mc49g8hZzNNVX4Yh4rDNy9W9qMWy5uKJuOIurBzn+YlyiigoQKQkrW", + "01CwjdN0PSr6lb+cp7WN37bRWx7p8t05+ec4XKujv5z+7d3/lRcPf9t59/z6+r/nz/52+oL+93V88fKL", + "Qk5WB+591+i7lWdq4GWsRN21RY9zrEKP4jPjUjVAzb5BiqNEf9xHJ2CgHQ1ZDz2niggcH6FhgFPat8Ds", + "hzwZBqhD3uNQma8QZ0h3ZY+Ot/THFybsRn/80dmAn+p9RPaMWFgg5+EcMhtHPMGUbQ3ZkNm+kFuIhEMb", + "/RShEKcqE0TviNY14wUaCxwWZ8PF4F30Eafpp60hA0uUvFdCryDFQuVRvm4E2Gg7K3MoZJuTCM1xnBFp", + "Ldkhy+UHmOa6E4XFlKh+7kIER03tYKYBKF4zg4tqbMPhoOvZR6Tb6Y2MqVSEodwrQSUgL+q4IJXDQYX8", + "DweH688fcxxagX6A3cv3ah1StqAPg8AwtGHGo5lS6frwBeA3hkbQL69fX2gw6H8vkeuogEW+xcYYw2ka", + "UyLNqZqKQSexcUFbge/kzOxuywW9No31Z3GLMIwnMDB6/fwSKSISygz/7oQanBMa6vXB+Q6VMtOoSDE6", + "Pjl/stVvcTEYYJvPf8U+vs5XWDtGcM6tZQsTviic5hq+XXR22tXqlKXQQtGCc9OnXKDYMJiCro/QlSTV", + "KAbYKnPEY3YyXhQeMsPVh8GW6zGtc4oj9CrX73A+lfz2QYEMrsuCLqFbG9hiDnWXeu9W5wrH1dZ+sawN", + "jnCxQtbpDaK4mRWsJn8PxIHmOav7Hjej7bLTUg/mR41i77+5BrK3qS25aSR3NSitFISYB3N/3yjsz4mp", + "djv07OIKIpexHEmGUznjqjk4AyPXBpH3VCq5HMfWKpxgOYa7Kp5MdPaKwMCvGY0tMsYgMqK+jK8eZ/09", + "Yw1+vBjvlVHZXxpabRW0bxRZ3cgQfFHJVd5gfv66MdLfZDqVaGcfMyjLMRcI9tkBzt2AeoJgjqWkU0Yi", + "dHZR3PorHB6u+9qaHu32dw4O+zuDQX9n0Mb9k+BwxdjnxyftBx/sGoP4CI+PwuiITL7A/WQR2ygcOL7F", + "C4mGTiUcBkYHLSmfJbK1amOro73lOPLPCxuvC8F1geGbBIK3i/BecR3/snoRv7Ve8eCfX3Rnn7QVw5fQ", + "2H012sQxSlDIszhif1ZorCnPmAIkshaLJKrIcQDEesVuGL9l1aUb/5im33cZEQt0fX5e8aYKMrHXvVss", + "nKdp4z7wdKNt2F2j3q2dTSnY+i4CrOucsCSBvno4ddn14+I6DNa1cAEV6p/3mJQyA2699yvWVDPeIzIf", + "ZZlP0dGvXITm1dXZaWXDMT7YORwcPuodjncOevvRYKeHd/YOersP8GCyFz7ca0iM0j5M4vMjH6oU2hwR", + "DYAHR5gJYo+ONA3loQvjTKH8kpomzhOtMaKSHmrif8E2fWVUUt0DSNdQv4kXuaq68uMLrAnVfZvCX6u/", + "uJxlSqtB8I2cZQrpv2DKeglW1V/dhaH5I/SCwzd2pl0tKGs2g2mOWTReLDev2xcdGwEiiFRckAgGswzs", + "CD3NmVbO9iyb60hiHw0vtZFSEAW2ZQxqq97b3Qq6gYV60A0MCINu4CCjH80K4QkmH3QDOxFvkGUZb3yO", + "YoJj4GFFEEamaEw/GJLTU6dS0dCYWBh2s4ns7E02Eo2MCG06yjEn+1bM5h85qr4+Rx24d/AXZC0w/ddW", + "fuxTJqH93Uf7jw4e7j46aBW1WExwPTc+gbiT5cmtZc1hmo1cgqiGpZ9cXIHw0YJNZokJk7RrL2w3zThC", + "re1RhoqMU8Xgj/qPysGaEc/GccnTYKO1ISKwTXqwhnOOdzSe08mEvfsQ3uz+Jmiy8/5A7o69xlE+kF+T", + "PCt7x5bMLjLumavG/ng6QCghG0NOXxEJK0CXRCHAn55mWFqi5uEiFuVcYKqFuBex9vf29g4fPththVd2", + "diXCGYH9tzzLczuDEolBS9R5dXmJtksIZ/p0MXSpIFIvztyi8NIZGmaDwR5Bg0p4nbY99nxY0qCwFFhj", + "+54njSC/thqLXZQFOkS95NrMEpV7ob23N3i4/+DwQTsythbPSLxfzWFsO3taLEhI6Lyy8x3wqL4+vkC6", + "dzHBYVXD39nd239w8PBwo1mpjWalBGYyoUptNLHDhwcP9vd2d9rFTvu8pvZWQIVgq7zLQ3QepPDshgcU", + "y6y32yQtfFricqjdyui+IlywHhu2STBocROMSuiVluIQUUcrUWWFtHSbaauNn8HPIvU4TWkntbrYNk5z", + "dVjmBVazMzbhy27xTQw+G+ziDiFSrfhISMgVEUZJ5HhXbvlZXQrCZ2JJUJQRCzmjGwlsAY7N0UCK1QyU", + "VfiQsmk1cHhpwDZmmJnD6nt/MK5t2MZjJP0BGq9FBrAyDl2JcBGq0co7TeXIb1UsdyzINIuxQPVY5BVT", + "loskpuymTe9ykYx5TEOkP6ib8xMex/x2pF/Jv8JatlqtTn8wKk4la+a5mZw9kzYbUhu3WMJf9Sq3alEu", + "IPm3zffbkFe4jQPOG637VBtvJlz3itH3JUSvXqLZ3x00BTU1dFoJZ1oO9d6Ut1uU9VG8i8I+zpNNeI7E", + "zGlNzYKt6sGV9fpWC6daq0K4ljUB1HE+PXdJqQrX0mWhVoK43cFa3XvtZrMtSVgdff/wwcODlre1vkjV", + "XpF59QsU63myQqFu2KnzNlrb4YPDR4/29h882t1IP3IHHQ3703TYUd6fWk6Zms72YAD/22hS5qjDP6WG", + "447qhCr5YT57Qp9WkG5xgaLB6l6V9bzYSWfmVxXwdiruCm3puKJyldKEdchkQsBxNDJw6xWTqQX0tJpD", + "iFMcUrXwWID4FmIcUN6kdhGgRe+1yXpAavtGeKKIgNMImY2Lm3AdNzj6T2PZ1XDhsPWlT5mNm6zIl/VR", + "jQ1pgoKimoeihYPAYITvBPw2Bya6xbLi1dfPoSJRt5QGrn78Y1q0z3LrcD1PdFscbPsus/iT2pa3v7ad", + "JaujoiTXIb5KhDaToNYIIOKojYPdI5E9N2TC9REUNf5gBeDnfTUal69jr7zvXrm7XUjdzcdtl3hv+Tsj", + "wTYfr3SCv8mH9ZupgI92DhbkRd/dCkr4sMmcrzSlPUlcOZDaxVVqEqzb20mo1Bh1SJKqhYvAd5bp1mbn", + "Pcd5h15k/MoxU4NHXyNq+2plmPb/kEQ65SM2N8jaw7WlPW2MjfSrq6f18BVjE9pEAtVwi9r1aKlW1BFY", + "VbPGFI8Bg8/GJU+z+kWqDerUNJn4BeW4AgGuUM06y3WlP620stJMmvfGnK9+YVEfKl01n88EmTW/1gf6", + "mjMqbQD36pkmzD1VQcGeswAygNUgyE30ZT/A6rCPc/w+HwGsZSxRLTefWUcpz+2zx3D3/JXLOEAnrguY", + "Rj3L4uMvq3bksGp5M1aVP3In+F7Cs/xnBUdroq0achZjdFdXWNKsi4SZoGpxqQWCDU4jWBBxnBk0BEkB", + "i4Cfi8Eh2P3TJzBTJx5t9RlhRNAQHV+cAZYkmOGp3rLrcxTTCQkXYUxsrPLS2S5ctX95ctYzlyzyZHhQ", + "vEABQFwWquOLM0iAY8sGBIP+bh9y/fKUMJzS4CjY6+9Aih8NBljiNtxhg0friNJ0CJLsLLIS97FpokEr", + "U86kAc7uYFArQ4GLJCPbv0njYTHitbVSaOr8LMdbLIXgOk3ATv9TN9gf7Gw0n7V5QXzDXjGcqRkX9AOB", + "aT7YEAifNegZM1a1y0RMbMMCZ4OjX6vY+uubT2+6gcySBGsV0YCrgFXKZZMKQyTCiJFbe7nxNz7uo0tj", + "k0CSkKKCmnEZkEizJIwUFv3pB4RFOKNzMmSWE5scL1jATY4EaQ5s4uiraGaGNrtvSJhI9ZhHixp08+62", + "dXegjVQBvHF9jjxhYdpQqMPHHU1eJBlyb0IowjBTRZodkxDphsAh5oS+98bCQ3yv39t9mr9zFV2qvF2r", + "u5SFcRYVArBaScN7x9pUhLApnm6IR194Bi3s/Muh0E7SMB4RE9aaLtSMM/OcjTOmMvM8FvxWEqHlkb2S", + "YcGizea8EpfJn0cTuBZhLnHqMbfNFLc/3pDFp/6QHUeJu3Rr07jiWHKb+8oEKFCJ8mTCQ+bVoOUI635G", + "Y1dSrKaoEuhqGGhROQz081RgrZJlcoZwCAEJ+scycDoGm7kAcbdVn2uIGUp5msVaeYDtMcmxKn3A7TYc", + "x0gB/rhvtRAFmDSsR5JQEJ+t9LfLly8Q8E8orQLNiuhyWANlWvrlSWL1gP0he4LDGTKCEZInDgMaDYOi", + "hMYWCLFMEiObej2QrH+F2kJmmC6N/trv666M0D5Cv340vRxprEmTkeI3hA2DT11UejGlapaN83dvGhbc", + "4Ku5rKA86hiGtOXuA+sVlnizYWaYRYhbBhAvEEYFrZVNsjFlWCya6tHwTDXHu5jr0rZZcZfvYDDYWn+e", + "YZfqUVcqDTWmflqSzrtfTTBZobwsmEq157QYYPYufGTE8R1Ixsc4cle0fqoAa1QAa7uUhDt8bxXA7Y80", + "+mTQNyYmvrImoaFEkZPQKRY4IQqSVP/qx3kILaX6b3f6CL4GY8lXkbdbAk9doX+zhNj7jbWf8ipKgAv7", + "d4B/MG6RoQzGfXRX4+LY5MfN61HeK3SEzXKI2PVbH8+I+hEwbnBXrNQlUvyO+Htf8OcZsSpSAbQaN9uG", + "zPRl07Z+BUIQnEjbi2msbZlLmFPvkjCFoOqg7Nt/nZoN0eVvYz59e4QMCGNbc1Ha1Hi5D1gLRQtL+Mhk", + "Dsm/swl1whlmUyJRx8jP3//1b1c37vd//dvWjfv9X/8Gct+2VVChu7zi4dsj9HdC0h6O6Zy4xUDEJJkT", + "sUB7A1uLA1550vPIIRuyV0Rlgsk83kivC2BiOgSVncF6KMuIRBJACImzJzYQxriYPCaeo2UDyjul6O6S", + "pWtXUFqAlooOB+BkkzKqKI4Rz5TJcQnzgEs5xUTMmoPy4HVv2ZL/dD1/UeS9MtjbMxPckMGYiqEeujNF", + "NE2fqHN5+WSrj0DdN1gBwU5gNxTdWEug/5MnredJhqNUGQpA2fCmUmbGRl/bqW1zF862pqyNzd42ASnm", + "ibZd3WJ+qt0tPG9+uDkvnM8VduoyiTf7wj5/vb6Coq1syq+3zw73lmFu0+QXIPse1iTq2AzHeSKTSi7+", + "74X0d8KASyUcci6MuEmfcmcWzglnk5iGCvXcXGyRydzqqSLIfWEHr+ysEXbrqkfol0XFdiXgrFFo5LFn", + "dyk9aoNuIkaKWwQFrv2UJOtQ55TKkOtvS9jSC3EKgLRALOi0jEXrfDun8HsuclYq5nnZV0eQd+flsUNn", + "rC4b7oApntYY4ndkhLU0H6V7N/cJm6/yXXRVU1Y4gX4s1BzcnRZ01w4hH5rfJ49QVAOb5oKzPLF4E3rZ", + "1OPfcKPtCJ6FXxLhqNpM1KSXKJZlPkXhjIQ3ZkG2uM8qjeDM1f/59nqAyZ++gfS30/8p7lsYjgWsVhmL", + "ZzbnyLezFWGEjUzFr3f8aBHMA2SI0hg7R6pJ54HlgoVbf6gTyDuRDPViPPeIki6yOHaO+DkRqsgiX+an", + "2x+1ftBCT3bUtlIXuXr1vEdYyCEmx4CuUSFxSaO/rrZsNsws5SeatLGvAFQOMZqV0S/YfxM6hfJsjn/a", + "fWrzOf5p96nJ6PinvWOT03HrmyHL4K5Y811rr/cY+bTySqtAA9ZkUjuv0/byVnei8Nkc+puofPkEf2p9", + "bbS+MrhWKn55OYNvqPrZLPHf55wgRzYftOGViz/7g6l8d+t6shhZKvxX8cXbxCZcFJnZbdmw+xcgR3OM", + "K/Pflj7UgiBXagcOdc9OuzbpvkmVnweI35FH1c3jzrVEO+7du1OPkzGdZjyT5YB2qLFAZFGQtsKA75v+", + "WojnRg32B8bSwV2KjjtXUH/i/TdSnesbapi3rWG7Rnl2re5GeS6Oatprz26GP7XnVtpzCVyrtec8j+u3", + "VJ/NIN9Nf3b45gO4vcL8U4O+Cw1aZpMJDSlhqshBtBTVYlOY3cN7Jcw64Uun0RUm3FqDLpIrr1ZOLPJ+", + "j0iEfPC7V5xdorP7GR/LTUR85FTVQhg266o/Gj4M7pY5372Oep9R7Fm5np1fGzSXQ2I+XX81JO/J3YPw", + "3A0ZMlf87q1h6m9RjqhIcSRJTEKFbmc0nME9Ef0b9G+ukeA0fZtfDN06Qs8g/rR8VRUG70giKI4hYzqP", + "TbL/t/MkeXu0nDPi+vwcPjJXREx2iLdHyOWJyGlM6lblex96FTGWCr2wt1k6esMFj2OTnfmthmdpfVv2", + "Rkhxh3bIfLdDGLm1HdIJelu6KPK24aaIQ8Lnepe+E+V3m5Pjm7UojgQAztxZJyxquCWioea/I7Iz8KY+", + "anlfxUzjG19XWZrMcz7N8wtUUBmnaVv0tdMELJ4nyQocRp1SQQCpIp6pv0gVEWHq1VrsbkJu1MGh+UPh", + "G1NdtVJezpSg8IHK3r32giowNaRd5Qrz1zxJAlPrLsG+ShRffu+n3uGywah3pnS556fM2OTaTpXZl+7t", + "1CSHLYEC2Ua81uUr0+APr7m4WjHfGQ2/g6VXzIJCCRkWjRewt0URnvt1aQE2slgZyDu7Li+NuHeNNGJr", + "9/zhaaTAjz84lYRcQMFv6Qrw3Z/ospLFUSL3DlT8KippdZ3Ve31+vtVENKZmdCPJiJ/msA30/MPLFCiC", + "dv+oxdT/xPkCVjkLNUGoRhvd2ayVAoljnunel9KnQmEQuZCKJMZgn2Qx3LyDsHqbwACXC590EVUS0nB3", + "wWVVKnoxZGMy0fIwJUKPrT+H9GyF7eEzay8Vzsn3wtDgj2HXQkZVMOWwaoJarbpImrpkqj7bKc//+tlT", + "egqGarXwikSdmN6YaoJoLlGsH7ZWWrqmKsvXTs/w+ZSV1x3yXbs1OJsj8x+Bw53V2Jqrq3nv2NozUiYW", + "x39go/1sTa7la2LDwpQOdqUClf0hOydK6DZYEBTyOIZ6BEZ/304FD7ehaF6Y0shUz4PJAcNrfp3AiCcX", + "V9DOpIDvDpn+Y7lsW32irvrb2fbLNb4/U7Dzf7CeYxa4iiz8G/7TrbP5UUAjDckGEuXpKk2cpz8VcVuH", + "96fZei/NVjiLzVfTmQocglIsbaVlv4lqy5NtfzQPZ+tO9BUOZ9euWsSPoe3a5PLrhnELvBdEadcUEZMW", + "4O5pkuf5/+/p1S8NOLcEUGLKsQl+KWDqivzRsPvrx8mV4bhRlNyd0pZLufHD0NZdSz47BxeoVobHfSFz", + "g2luJZAAvex9EuUCZyttM1d/Cqrt5aqlq7vWLZf/Mxk+cx9SUTcmrzTWH7K8tJrLMKqtq64zrVBE5Y3p", + "wVpPfeSvgGfsPFsGb8gURyGOQ5N3Pi8FZ8o3ygbr61WpPOI3o7diEM9G5zXwZF6y7D6ZHH6cgN0r10QD", + "jLPq1Mr49Gvb5i6i060w2yA23a3gZ2R6i8j0ErDaVGAxBe0st7KVyPLyGVANqt9QSCVXSr5dXPtnyOuv", + "hx4OTxul9c+I9jtTCIoroWen9z+MvUxzFR69ra2Cni1vVHYNraJgC6JUkJ6r/xIZgFl4GFujXj2pP2Sv", + "Z8T9hagLpSSRraAfLxBlUPDGFcH7s0SCc1VU2G+usmRI5KngybFdzRrjpXU5SN9BzMb5KrqeEng0yZK8", + "WPyzx1D+WpjIPjTBNIa4UgdS8j4kJJKAk1v1MpPeUL+8nuTaWa6I0cwLSYWZVDxxe392ijo4U7w3JUzv", + "RVGzKRV8TqN6zeBKvU7fbMFC/ApG2vQDTaukt7bezTLhVfEW5UWqbMGdAj/d7gQ/xUQ9w7DebW3kOSAq", + "zlGMxZRs/RQl91mUlL1JTm5UJEq7C1HtHEwt/T7f4jJU7ny826tQ1z+OT6SUkfUeJgyY50Zf0x2sHwsF", + "B3cnH+767tX1PfahPyPOwC3du4IOdI8+hHnOQxyjiMxJzFMoRW3aBt0gE7EtrHu0vR3rdjMu1dHh4HAQ", + "fHrz6f8HAAD//wgwYbc14gAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 2c88ea0..3ea7cf8 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -105,22 +105,12 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima maxMemoryPerInstance = int64(memSize) } - // Parse max total memory (empty or "0" means unlimited) - var maxTotalMemory int64 - if cfg.MaxTotalMemory != "" && cfg.MaxTotalMemory != "0" { - var memSize datasize.ByteSize - if err := memSize.UnmarshalText([]byte(cfg.MaxTotalMemory)); err != nil { - return nil, fmt.Errorf("failed to parse MAX_TOTAL_MEMORY '%s': %w", cfg.MaxTotalMemory, err) - } - maxTotalMemory = int64(memSize) - } - + // Note: Aggregate CPU/memory limits are now handled via oversubscription ratios + // in the ResourceManager, wired up via SetResourceValidator after initialization. limits := instances.ResourceLimits{ MaxOverlaySize: int64(maxOverlaySize), MaxVcpusPerInstance: cfg.MaxVcpusPerInstance, MaxMemoryPerInstance: maxMemoryPerInstance, - MaxTotalVcpus: cfg.MaxTotalVcpus, - MaxTotalMemory: maxTotalMemory, } meter := otel.GetMeterProvider().Meter("hypeman") diff --git a/lib/resources/disk.go b/lib/resources/disk.go index 6ea0bdd..2b6bf76 100644 --- a/lib/resources/disk.go +++ b/lib/resources/disk.go @@ -126,3 +126,42 @@ func parseDiskIOLimit(limit string) (int64, error) { return int64(ds.Bytes()), nil } + +// DiskIOResource implements Resource for disk I/O bandwidth tracking. +type DiskIOResource struct { + capacity int64 // bytes per second + instanceLister InstanceLister +} + +// NewDiskIOResource creates a disk I/O resource with the given capacity. +func NewDiskIOResource(capacity int64, instLister InstanceLister) *DiskIOResource { + return &DiskIOResource{capacity: capacity, instanceLister: instLister} +} + +// Type returns the resource type. +func (d *DiskIOResource) Type() ResourceType { + return ResourceDiskIO +} + +// Capacity returns the total disk I/O capacity in bytes per second. +func (d *DiskIOResource) Capacity() int64 { + return d.capacity +} + +// Allocated returns total disk I/O allocated across all active instances. +func (d *DiskIOResource) Allocated(ctx context.Context) (int64, error) { + if d.instanceLister == nil { + return 0, nil + } + instances, err := d.instanceLister.ListInstanceAllocations(ctx) + if err != nil { + return 0, err + } + var total int64 + for _, inst := range instances { + if isActiveState(inst.State) { + total += inst.DiskIOBps + } + } + return total, nil +} diff --git a/lib/resources/resource.go b/lib/resources/resource.go index 39e7e55..939fdcb 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -7,6 +7,7 @@ import ( "fmt" "sync" + "github.com/c2h5oh/datasize" "github.com/kernel/hypeman/cmd/api/config" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/paths" @@ -20,6 +21,7 @@ const ( ResourceMemory ResourceType = "memory" ResourceDisk ResourceType = "disk" ResourceNetwork ResourceType = "network" + ResourceDiskIO ResourceType = "disk_io" ) // SourceType identifies how a resource capacity was determined. @@ -62,6 +64,7 @@ type AllocationBreakdown struct { DiskBytes int64 `json:"disk_bytes"` NetworkDownloadBps int64 `json:"network_download_bps"` // External→VM NetworkUploadBps int64 `json:"network_upload_bps"` // VM→External + DiskIOBps int64 `json:"disk_io_bps"` // Disk I/O bandwidth } // DiskBreakdown shows disk usage by category. @@ -78,6 +81,7 @@ type FullResourceStatus struct { Memory ResourceStatus `json:"memory"` Disk ResourceStatus `json:"disk"` Network ResourceStatus `json:"network"` + DiskIO ResourceStatus `json:"disk_io"` DiskDetail *DiskBreakdown `json:"disk_breakdown,omitempty"` GPU *GPUResourceStatus `json:"gpu,omitempty"` // nil if no GPU available Allocations []AllocationBreakdown `json:"allocations"` @@ -99,6 +103,7 @@ type InstanceAllocation struct { VolumeOverlayBytes int64 // Sum of volume overlay sizes NetworkDownloadBps int64 // Download rate limit (external→VM) NetworkUploadBps int64 // Upload rate limit (VM→external) + DiskIOBps int64 // Disk I/O rate limit (bytes/sec) State string // Only count running/paused/created instances VolumeBytes int64 // Sum of attached volume base sizes (for per-instance reporting) } @@ -210,23 +215,36 @@ func (m *Manager) Initialize(ctx context.Context) error { } m.resources[ResourceNetwork] = net + // Discover disk I/O (reuses existing DiskIOCapacity method) + diskIO := NewDiskIOResource(m.DiskIOCapacity(), m.instanceLister) + m.resources[ResourceDiskIO] = diskIO + return nil } // GetOversubRatio returns the oversubscription ratio for a resource type. +// Returns 1.0 (no oversubscription) if the config value is not set or <= 0. func (m *Manager) GetOversubRatio(rt ResourceType) float64 { + var ratio float64 switch rt { case ResourceCPU: - return m.cfg.OversubCPU + ratio = m.cfg.OversubCPU case ResourceMemory: - return m.cfg.OversubMemory + ratio = m.cfg.OversubMemory case ResourceDisk: - return m.cfg.OversubDisk + ratio = m.cfg.OversubDisk case ResourceNetwork: - return m.cfg.OversubNetwork + ratio = m.cfg.OversubNetwork + case ResourceDiskIO: + ratio = m.cfg.OversubDiskIO default: return 1.0 } + // Default to 1.0 (no oversubscription) if not configured + if ratio <= 0 { + return 1.0 + } + return ratio } // GetStatus returns the current status of a specific resource type. @@ -296,6 +314,11 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error return nil, err } + diskIOStatus, err := m.GetStatus(ctx, ResourceDiskIO) + if err != nil { + return nil, err + } + // Get disk breakdown var diskBreakdown *DiskBreakdown m.mu.RLock() @@ -331,6 +354,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error DiskBytes: inst.OverlayBytes + inst.VolumeBytes, NetworkDownloadBps: inst.NetworkDownloadBps, NetworkUploadBps: inst.NetworkUploadBps, + DiskIOBps: inst.DiskIOBps, }) } } @@ -345,6 +369,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error Memory: *memStatus, Disk: *diskStatus, Network: *netStatus, + DiskIO: *diskIOStatus, DiskDetail: diskBreakdown, GPU: gpuStatus, Allocations: allocations, @@ -360,6 +385,81 @@ func (m *Manager) CanAllocate(ctx context.Context, rt ResourceType, amount int64 return amount <= status.Available, nil } +// ValidateAllocation checks if the requested resources can be allocated. +// Returns nil if allocation is allowed, or a detailed error describing +// which resource is insufficient and the current capacity/usage. +// Parameters match instances.AllocationRequest to implement instances.ResourceValidator. +func (m *Manager) ValidateAllocation(ctx context.Context, vcpus int, memoryBytes int64, networkDownloadBps int64, networkUploadBps int64, diskIOBps int64, needsGPU bool) error { + // Check CPU + if vcpus > 0 { + status, err := m.GetStatus(ctx, ResourceCPU) + if err != nil { + return fmt.Errorf("check CPU capacity: %w", err) + } + if int64(vcpus) > status.Available { + return fmt.Errorf("insufficient CPU: requested %d vCPUs, but only %d available (currently allocated: %d, effective limit: %d with %.1fx oversubscription)", + vcpus, status.Available, status.Allocated, status.EffectiveLimit, status.OversubRatio) + } + } + + // Check Memory + if memoryBytes > 0 { + status, err := m.GetStatus(ctx, ResourceMemory) + if err != nil { + return fmt.Errorf("check memory capacity: %w", err) + } + if memoryBytes > status.Available { + return fmt.Errorf("insufficient memory: requested %s, but only %s available (currently allocated: %s, effective limit: %s with %.1fx oversubscription)", + datasize.ByteSize(memoryBytes).HR(), datasize.ByteSize(status.Available).HR(), datasize.ByteSize(status.Allocated).HR(), datasize.ByteSize(status.EffectiveLimit).HR(), status.OversubRatio) + } + } + + // Check Network (use max of download/upload since they share physical link) + netBandwidth := networkDownloadBps + if networkUploadBps > netBandwidth { + netBandwidth = networkUploadBps + } + if netBandwidth > 0 { + status, err := m.GetStatus(ctx, ResourceNetwork) + if err != nil { + return fmt.Errorf("check network capacity: %w", err) + } + if netBandwidth > status.Available { + return fmt.Errorf("insufficient network bandwidth: requested %s/s, but only %s/s available (currently allocated: %s/s, effective limit: %s/s with %.1fx oversubscription)", + datasize.ByteSize(netBandwidth).HR(), datasize.ByteSize(status.Available).HR(), datasize.ByteSize(status.Allocated).HR(), datasize.ByteSize(status.EffectiveLimit).HR(), status.OversubRatio) + } + } + + // Check Disk I/O + if diskIOBps > 0 { + status, err := m.GetStatus(ctx, ResourceDiskIO) + if err != nil { + return fmt.Errorf("check disk I/O capacity: %w", err) + } + if diskIOBps > status.Available { + return fmt.Errorf("insufficient disk I/O: requested %s/s, but only %s/s available (currently allocated: %s/s, effective limit: %s/s with %.1fx oversubscription)", + datasize.ByteSize(diskIOBps).HR(), datasize.ByteSize(status.Available).HR(), + datasize.ByteSize(status.Allocated).HR(), datasize.ByteSize(status.EffectiveLimit).HR(), + status.OversubRatio) + } + } + + // Check GPU if needed + if needsGPU { + gpuStatus := GetGPUStatus() + if gpuStatus == nil { + return fmt.Errorf("insufficient GPU: no GPU available on this host") + } + availableSlots := gpuStatus.TotalSlots - gpuStatus.UsedSlots + if availableSlots <= 0 { + return fmt.Errorf("insufficient GPU: all %d %s slots are in use", + gpuStatus.TotalSlots, gpuStatus.Mode) + } + } + + return nil +} + // CPUCapacity returns the raw CPU capacity (number of vCPUs). func (m *Manager) CPUCapacity() int64 { m.mu.RLock() diff --git a/openapi.yaml b/openapi.yaml index c53992f..d87dedd 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1058,6 +1058,11 @@ components: format: int64 description: Upload bandwidth limit in bytes/sec (VM→external) example: 125000000 + disk_io_bps: + type: integer + format: int64 + description: Disk I/O bandwidth limit in bytes/sec + example: 104857600 Resources: type: object @@ -1071,6 +1076,8 @@ components: $ref: "#/components/schemas/ResourceStatus" network: $ref: "#/components/schemas/ResourceStatus" + disk_io: + $ref: "#/components/schemas/ResourceStatus" disk_breakdown: $ref: "#/components/schemas/DiskBreakdown" gpu: @@ -1304,6 +1311,12 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" + 409: + description: Conflict - insufficient resources or name already exists + content: + application/json: + schema: + $ref: "#/components/schemas/Error" 500: description: Internal server error content: