diff --git a/internal/api/commitment.go b/internal/api/commitment.go index ac1d5366c..714adfd7a 100644 --- a/internal/api/commitment.go +++ b/internal/api/commitment.go @@ -431,7 +431,7 @@ func (p *v1Provider) CanConfirmNewProjectCommitment(w http.ResponseWriter, r *ht InfoVersion: serviceInfo.Version, ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ dbProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain, *serviceInfo), + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain), ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ loc.ResourceName: { TotalConfirmedBefore: totalConfirmed, @@ -541,85 +541,90 @@ func (p *v1Provider) CreateProjectCommitment(w http.ResponseWriter, r *http.Requ return } dbCommitment.NotifyOnConfirm = req.NotifyOnConfirm + var auditEvents []audittools.Event - // we do an information to liquid in any case, right now we only check the result when confirming immediately - newStatus := liquid.CommitmentStatusPlanned - totalConfirmedAfter := totalConfirmed if confirmBy.IsNone() { - newStatus = liquid.CommitmentStatusConfirmed - totalConfirmedAfter += req.Amount - } - ccr := liquid.CommitmentChangeRequest{ - AZ: loc.AvailabilityZone, - InfoVersion: serviceInfo.Version, - ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ - dbProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain, *serviceInfo), - ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ - loc.ResourceName: { - TotalConfirmedBefore: totalConfirmed, - TotalConfirmedAfter: totalConfirmedAfter, - // TODO: change when introducing "guaranteed" commitments - TotalGuaranteedBefore: 0, - TotalGuaranteedAfter: 0, - Commitments: []liquid.Commitment{ - { - UUID: dbCommitment.UUID, - OldStatus: None[liquid.CommitmentStatus](), - NewStatus: Some(newStatus), - Amount: req.Amount, - ConfirmBy: confirmBy, - ExpiresAt: req.Duration.AddTo(confirmBy.UnwrapOr(now)), - }, - }, - }, - }, - }, - }, - } - commitmentChangeResponse, err := datamodel.DelegateChangeCommitments(r.Context(), p.Cluster, ccr, *loc, *serviceInfo, p.DB) - if respondwith.ObfuscatedErrorText(w, err) { - return - } - - resourceInfo := core.InfoForResource(*serviceInfo, loc.ResourceName) - - if ccr.RequiresConfirmation() { - // if not planned for confirmation in the future, confirm immediately (or fail) - if commitmentChangeRequestWasRejected(commitmentChangeResponse, w, true) { - return - } - dbCommitment.ConfirmedAt = Some(now) - dbCommitment.Status = liquid.CommitmentStatusConfirmed - - // handle public transfer commitments (does not alter the confirmed commitment) + // When the commitment is to be confirmed immediately, the capacity check + // is carried out together with the transferability check in the cache. mailTemplate := None[core.MailTemplate]() if mailConfig, exists := p.Cluster.Config.MailNotifications.Unpack(); exists { mailTemplate = Some(mailConfig.Templates.TransferredCommitments) } - transferableCommitmentCache, err := datamodel.NewTransferableCommitmentCache(tx, *serviceInfo, *loc, now, p.generateProjectCommitmentUUID, p.generateTransferToken, mailTemplate) - if respondwith.ObfuscatedErrorText(w, err) { - return - } - err = transferableCommitmentCache.CheckAndConsume(dbCommitment, totalConfirmed) + transferableCommitmentCache, err := datamodel.NewTransferableCommitmentCache(tx, p.Cluster, *serviceInfo, *loc, now, p.generateProjectCommitmentUUID, p.generateTransferToken, mailTemplate) if respondwith.ObfuscatedErrorText(w, err) { return } - ae, err := transferableCommitmentCache.GenerateAuditEventsAndMails(p.Cluster.BehaviorForResourceLocation(*loc).IdentityInV1API, audit.Context{ + auditContext := audit.Context{ UserIdentity: token, Request: r, - }) + } + result, err := transferableCommitmentCache.CanConfirmWithTransfers(r.Context(), dbCommitment, *dbProject, *dbDomain, true, false, auditContext, cadf.CreateAction) if respondwith.ObfuscatedErrorText(w, err) { return } - for _, event := range ae { - p.auditor.Record(event) + if commitmentChangeRequestWasRejected(result, w, true) { + return + } + + // retrieve mails and audit event + auditEvents = append(auditEvents, transferableCommitmentCache.RetrieveAuditEvents()...) + err = transferableCommitmentCache.GenerateTransferMails(p.Cluster.BehaviorForResourceLocation(*loc).IdentityInV1API) + if respondwith.ObfuscatedErrorText(w, err) { + return } + + dbCommitment.ConfirmedAt = Some(now) + dbCommitment.Status = liquid.CommitmentStatusConfirmed } else { - // TODO: when introducing guaranteed, the customer can choose via the API signature whether he wants to create - // the commitment only as guaranteed (RequestAsGuaranteed). If this request then fails, the customer could - // resubmit it and get a planned commitment, which might never get confirmed. + // when the commitment is not to be confirmed immediately, we check + // (or inform the liquid) about the capacity independently. + ccr := liquid.CommitmentChangeRequest{ + AZ: loc.AvailabilityZone, + InfoVersion: serviceInfo.Version, + ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ + dbProject.UUID: { + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain), + ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ + loc.ResourceName: { + TotalConfirmedBefore: totalConfirmed, + TotalConfirmedAfter: totalConfirmed, + // TODO: change when introducing "guaranteed" commitments + TotalGuaranteedBefore: 0, + TotalGuaranteedAfter: 0, + Commitments: []liquid.Commitment{ + { + UUID: dbCommitment.UUID, + OldStatus: None[liquid.CommitmentStatus](), + NewStatus: Some(liquid.CommitmentStatusPlanned), + Amount: req.Amount, + ConfirmBy: confirmBy, + ExpiresAt: req.Duration.AddTo(confirmBy.UnwrapOr(now)), + }, + }, + }, + }, + }, + }, + } + commitmentChangeResponse, err := datamodel.DelegateChangeCommitments(r.Context(), p.Cluster, ccr, *loc, *serviceInfo, tx) + if respondwith.ObfuscatedErrorText(w, err) { + return + } + if ccr.RequiresConfirmation() && commitmentChangeRequestWasRejected(commitmentChangeResponse, w, true) { + return + } + // TODO: change when introducing "guaranteed" commitments dbCommitment.Status = liquid.CommitmentStatusPlanned + + auditEvents = append(auditEvents, audit.CommitmentEventTarget{ + CommitmentChangeRequest: ccr, + }.ReplicateForAllProjects(audittools.Event{ + Time: now, + Request: r, + User: token, + ReasonCode: http.StatusCreated, + Action: cadf.CreateAction, + }, None[cadf.Action](), None[liquid.ProjectUUID]())...) } // create commitment @@ -627,22 +632,10 @@ func (p *v1Provider) CreateProjectCommitment(w http.ResponseWriter, r *http.Requ if respondwith.ObfuscatedErrorText(w, err) { return } - err = tx.Commit() if respondwith.ObfuscatedErrorText(w, err) { return } - - commitment := datamodel.ConvertCommitmentToDisplayForm(dbCommitment, loc.AvailabilityZone, p.Cluster.BehaviorForResourceLocation(*loc).IdentityInV1API, datamodel.CanDeleteCommitment(token, dbCommitment, p.timeNow), resourceInfo.Unit) - auditEvents := audit.CommitmentEventTarget{ - CommitmentChangeRequest: audit.EnsureLiquidProjectMetadata(ccr, *dbProject, *dbDomain, *serviceInfo), - }.ReplicateForAllProjects(audittools.Event{ - Time: now, - Request: r, - User: token, - ReasonCode: http.StatusCreated, - Action: cadf.CreateAction, - }) for _, event := range auditEvents { p.auditor.Record(event) } @@ -656,6 +649,9 @@ func (p *v1Provider) CreateProjectCommitment(w http.ResponseWriter, r *http.Requ } } + // render response + resourceInfo := core.InfoForResource(*serviceInfo, loc.ResourceName) + commitment := datamodel.ConvertCommitmentToDisplayForm(dbCommitment, loc.AvailabilityZone, p.Cluster.BehaviorForResourceLocation(*loc).IdentityInV1API, datamodel.CanDeleteCommitment(token, dbCommitment, p.timeNow), resourceInfo.Unit) respondwith.JSON(w, http.StatusCreated, map[string]any{"commitment": commitment}) } @@ -840,7 +836,7 @@ func (p *v1Provider) MergeProjectCommitments(w http.ResponseWriter, r *http.Requ InfoVersion: serviceInfo.Version, ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ dbProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain, serviceInfo), + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain), ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ loc.ResourceName: { TotalConfirmedBefore: totalConfirmed, @@ -868,14 +864,14 @@ func (p *v1Provider) MergeProjectCommitments(w http.ResponseWriter, r *http.Requ c := datamodel.ConvertCommitmentToDisplayForm(dbMergedCommitment, loc.AvailabilityZone, p.Cluster.BehaviorForResourceLocation(loc).IdentityInV1API, datamodel.CanDeleteCommitment(token, dbMergedCommitment, p.timeNow), resourceInfo.Unit) auditEvents := audit.CommitmentEventTarget{ - CommitmentChangeRequest: audit.EnsureLiquidProjectMetadata(ccr, *dbProject, *dbDomain, serviceInfo), + CommitmentChangeRequest: ccr, }.ReplicateForAllProjects(audittools.Event{ Time: p.timeNow(), Request: r, User: token, ReasonCode: http.StatusAccepted, Action: cadf.UpdateAction, - }) + }, None[cadf.Action](), None[liquid.ProjectUUID]()) for _, event := range auditEvents { p.auditor.Record(event) } @@ -1014,7 +1010,7 @@ func (p *v1Provider) RenewProjectCommitments(w http.ResponseWriter, r *http.Requ InfoVersion: serviceInfo.Version, ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ dbProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain, serviceInfo), + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain), ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ loc.ResourceName: { TotalConfirmedBefore: totalConfirmed, @@ -1052,14 +1048,14 @@ func (p *v1Provider) RenewProjectCommitments(w http.ResponseWriter, r *http.Requ c := datamodel.ConvertCommitmentToDisplayForm(dbRenewedCommitment, loc.AvailabilityZone, p.Cluster.BehaviorForResourceLocation(loc).IdentityInV1API, datamodel.CanDeleteCommitment(token, dbRenewedCommitment, p.timeNow), resourceInfo.Unit) auditEvents := audit.CommitmentEventTarget{ - CommitmentChangeRequest: audit.EnsureLiquidProjectMetadata(ccr, *dbProject, *dbDomain, serviceInfo), + CommitmentChangeRequest: ccr, }.ReplicateForAllProjects(audittools.Event{ Time: p.timeNow(), Request: r, User: token, ReasonCode: http.StatusAccepted, Action: cadf.UpdateAction, - }) + }, None[cadf.Action](), None[liquid.ProjectUUID]()) for _, event := range auditEvents { p.auditor.Record(event) } @@ -1132,7 +1128,7 @@ func (p *v1Provider) DeleteProjectCommitment(w http.ResponseWriter, r *http.Requ InfoVersion: serviceInfo.Version, ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ dbProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain, serviceInfo), + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain), ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ loc.ResourceName: { TotalConfirmedBefore: totalConfirmed, @@ -1167,14 +1163,14 @@ func (p *v1Provider) DeleteProjectCommitment(w http.ResponseWriter, r *http.Requ } auditEvents := audit.CommitmentEventTarget{ - CommitmentChangeRequest: audit.EnsureLiquidProjectMetadata(ccr, *dbProject, *dbDomain, serviceInfo), + CommitmentChangeRequest: ccr, }.ReplicateForAllProjects(audittools.Event{ Time: p.timeNow(), Request: r, User: token, ReasonCode: http.StatusNoContent, Action: cadf.DeleteAction, - }) + }, None[cadf.Action](), None[liquid.ProjectUUID]()) for _, event := range auditEvents { p.auditor.Record(event) } @@ -1300,7 +1296,7 @@ func (p *v1Provider) StartCommitmentTransfer(w http.ResponseWriter, r *http.Requ InfoVersion: serviceInfo.Version, ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ dbProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain, serviceInfo), + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain), ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ loc.ResourceName: { TotalConfirmedBefore: totalConfirmed, @@ -1330,8 +1326,8 @@ func (p *v1Provider) StartCommitmentTransfer(w http.ResponseWriter, r *http.Requ } ccr.ByProject[dbProject.UUID].ByResource[loc.ResourceName] = rcr cac[dbCommitment.UUID] = audit.CommitmentAttributeChangeset{ - OldTransferStatus: Some(dbCommitment.TransferStatus), - NewTransferStatus: Some(req.TransferStatus), + OldTransferStatus: dbCommitment.TransferStatus, + NewTransferStatus: req.TransferStatus, } dbCommitment.TransferStatus = req.TransferStatus @@ -1396,8 +1392,8 @@ func (p *v1Provider) StartCommitmentTransfer(w http.ResponseWriter, r *http.Requ } ccr.ByProject[dbProject.UUID].ByResource[loc.ResourceName] = rcr cac[transferCommitment.UUID] = audit.CommitmentAttributeChangeset{ - OldTransferStatus: Some(limesresources.CommitmentTransferStatusNone), - NewTransferStatus: Some(req.TransferStatus), + OldTransferStatus: limesresources.CommitmentTransferStatusNone, + NewTransferStatus: req.TransferStatus, } _, err = datamodel.DelegateChangeCommitments(r.Context(), p.Cluster, ccr, loc, serviceInfo, tx) @@ -1433,15 +1429,15 @@ func (p *v1Provider) StartCommitmentTransfer(w http.ResponseWriter, r *http.Requ c := datamodel.ConvertCommitmentToDisplayForm(dbCommitment, loc.AvailabilityZone, p.Cluster.BehaviorForResourceLocation(loc).IdentityInV1API, datamodel.CanDeleteCommitment(token, dbCommitment, p.timeNow), resourceInfo.Unit) auditEvents := audit.CommitmentEventTarget{ - CommitmentChangeRequest: audit.EnsureLiquidProjectMetadata(ccr, *dbProject, *dbDomain, serviceInfo), - CommitmentAttributeChangeset: cac, + CommitmentChangeRequest: ccr, + CommitmentAttributeChangesets: cac, }.ReplicateForAllProjects(audittools.Event{ Time: p.timeNow(), Request: r, User: token, ReasonCode: http.StatusAccepted, Action: cadf.UpdateAction, - }) + }, None[cadf.Action](), None[liquid.ProjectUUID]()) for _, event := range auditEvents { p.auditor.Record(event) } @@ -1631,8 +1627,8 @@ func (p *v1Provider) TransferCommitment(w http.ResponseWriter, r *http.Request) // check move is allowed cac := map[liquid.CommitmentUUID]audit.CommitmentAttributeChangeset{ dbCommitment.UUID: { - OldTransferStatus: Some(dbCommitment.TransferStatus), - NewTransferStatus: Some(limesresources.CommitmentTransferStatusNone), + OldTransferStatus: dbCommitment.TransferStatus, + NewTransferStatus: limesresources.CommitmentTransferStatusNone, }, } ccr := liquid.CommitmentChangeRequest{ @@ -1640,7 +1636,7 @@ func (p *v1Provider) TransferCommitment(w http.ResponseWriter, r *http.Request) InfoVersion: serviceInfo.Version, ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ sourceProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(sourceProject, sourceDomain, serviceInfo), + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(sourceProject, sourceDomain), ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ loc.ResourceName: { TotalConfirmedBefore: sourceTotalConfirmed, @@ -1662,7 +1658,7 @@ func (p *v1Provider) TransferCommitment(w http.ResponseWriter, r *http.Request) }, }, targetProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*targetProject, *targetDomain, serviceInfo), + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*targetProject, *targetDomain), ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ loc.ResourceName: { TotalConfirmedBefore: targetTotalConfirmed, @@ -1710,18 +1706,16 @@ func (p *v1Provider) TransferCommitment(w http.ResponseWriter, r *http.Request) resourceInfo := core.InfoForResource(serviceInfo, loc.ResourceName) c := datamodel.ConvertCommitmentToDisplayForm(dbCommitment, loc.AvailabilityZone, p.Cluster.BehaviorForResourceLocation(loc).IdentityInV1API, datamodel.CanDeleteCommitment(token, dbCommitment, p.timeNow), resourceInfo.Unit) - ccr = audit.EnsureLiquidProjectMetadata(ccr, sourceProject, sourceDomain, serviceInfo) - ccr = audit.EnsureLiquidProjectMetadata(ccr, *targetProject, *targetDomain, serviceInfo) auditEvents := audit.CommitmentEventTarget{ - CommitmentChangeRequest: ccr, - CommitmentAttributeChangeset: cac, + CommitmentChangeRequest: ccr, + CommitmentAttributeChangesets: cac, }.ReplicateForAllProjects(audittools.Event{ Time: p.timeNow(), Request: r, User: token, ReasonCode: http.StatusAccepted, Action: cadf.UpdateAction, - }) + }, None[cadf.Action](), None[liquid.ProjectUUID]()) for _, event := range auditEvents { p.auditor.Record(event) } @@ -1997,7 +1991,7 @@ func (p *v1Provider) ConvertCommitment(w http.ResponseWriter, r *http.Request) { InfoVersion: serviceInfo.Version, ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ dbProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain, serviceInfo), + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain), ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ sourceLoc.ResourceName: { TotalConfirmedBefore: sourceTotalConfirmed, @@ -2086,14 +2080,14 @@ func (p *v1Provider) ConvertCommitment(w http.ResponseWriter, r *http.Request) { c := datamodel.ConvertCommitmentToDisplayForm(convertedCommitment, targetLoc.AvailabilityZone, p.Cluster.BehaviorForResourceLocation(targetLoc).IdentityInV1API, datamodel.CanDeleteCommitment(token, convertedCommitment, p.timeNow), resourceInfo.Unit) auditEvents := audit.CommitmentEventTarget{ - CommitmentChangeRequest: audit.EnsureLiquidProjectMetadata(ccr, *dbProject, *dbDomain, serviceInfo), + CommitmentChangeRequest: ccr, }.ReplicateForAllProjects(audittools.Event{ Time: p.timeNow(), Request: r, User: token, ReasonCode: http.StatusAccepted, Action: cadf.UpdateAction, - }) + }, None[cadf.Action](), None[liquid.ProjectUUID]()) for _, event := range auditEvents { p.auditor.Record(event) } @@ -2192,7 +2186,7 @@ func (p *v1Provider) UpdateCommitmentDuration(w http.ResponseWriter, r *http.Req InfoVersion: serviceInfo.Version, ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ dbProject.UUID: { - ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain, serviceInfo), + ProjectMetadata: datamodel.LiquidProjectMetadataFromDBProject(*dbProject, *dbDomain), ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ loc.ResourceName: { TotalConfirmedBefore: totalConfirmed, @@ -2236,14 +2230,14 @@ func (p *v1Provider) UpdateCommitmentDuration(w http.ResponseWriter, r *http.Req c := datamodel.ConvertCommitmentToDisplayForm(dbCommitment, loc.AvailabilityZone, p.Cluster.BehaviorForResourceLocation(loc).IdentityInV1API, datamodel.CanDeleteCommitment(token, dbCommitment, p.timeNow), resourceInfo.Unit) auditEvents := audit.CommitmentEventTarget{ - CommitmentChangeRequest: audit.EnsureLiquidProjectMetadata(ccr, *dbProject, *dbDomain, serviceInfo), + CommitmentChangeRequest: ccr, }.ReplicateForAllProjects(audittools.Event{ Time: p.timeNow(), Request: r, User: token, ReasonCode: http.StatusOK, Action: cadf.UpdateAction, - }) + }, None[cadf.Action](), None[liquid.ProjectUUID]()) for _, event := range auditEvents { p.auditor.Record(event) } diff --git a/internal/api/commitment_test.go b/internal/api/commitment_test.go index 998e55cb3..215740856 100644 --- a/internal/api/commitment_test.go +++ b/internal/api/commitment_test.go @@ -878,34 +878,48 @@ func TestCommitmentLifecycleWithImmediateConfirmation(t *testing.T) { }.Check(t, s.Handler) } -// here, we only test a very basic case. The same code of the TransferableCommitmentCache +// We only test a very basic case. The same code of the TransferableCommitmentCache // is used by ScrapeCapacity, so the extensive testing of all the different edge cases -// happens there. This is only to prevent that we unintentionally break the integration with -// the API. +// happens there. This is only to prevent that we unintentionally break the API integration. func TestAutomaticCommitmentTransfer(t *testing.T) { s := setupCommitmentTest(t, testCommitmentsJSON) + // We modify the database so that the commitments for "first/capacity" go to the database for approval. + s.MustDBExec(`UPDATE resources SET handles_commitments = FALSE;`) // move clock forward past the min_confirm_date s.Clock.StepBy(14 * day) + // We create 2 commitments, one confirmed and one planned, to check that we calculate the missing amount correctly. + // The capacity is 10, overall confirmed is 3, other projects have a use of 4 and the commitment project has usage 2. + // If the commitment was not transferred, we would allocate 3 + 4 + 6 = 13 > 10. + // With the transfer, it works out as 4 + 6 = 10. dresden := s.GetProjectID("dresden") firstCapacityAZOne := s.GetAZResourceID("first", "capacity", "az-one") - uuid := s.Collector.GenerateProjectCommitmentUUID() - s.MustDBInsert(&db.ProjectCommitment{ + uuid1 := s.Collector.GenerateProjectCommitmentUUID() + uuid2 := s.Collector.GenerateProjectCommitmentUUID() + c := &db.ProjectCommitment{ CreatorUUID: "dummy", CreatorName: "dummy", CreationContextJSON: json.RawMessage(`{}`), ExpiresAt: s.Clock.Now().Add(time.Hour), Status: liquid.CommitmentStatusPlanned, - UUID: uuid, + UUID: uuid1, ProjectID: dresden, AZResourceID: firstCapacityAZOne, - Amount: 1, + Amount: 3, CreatedAt: s.Clock.Now(), Duration: must.Return(limesresources.ParseCommitmentDuration("1 hour")), TransferToken: Some(s.Collector.GenerateTransferToken()), TransferStatus: limesresources.CommitmentTransferStatusPublic, TransferStartedAt: Some(s.Clock.Now()), - }) + } + s.MustDBInsert(c) + + c.UUID = uuid2 + c.Status = liquid.CommitmentStatusConfirmed + c.ConfirmedAt = Some(s.Clock.Now()) + c.TransferToken = Some(s.Collector.GenerateTransferToken()) + s.MustDBInsert(c) + tr, _ := easypg.NewTracker(t, s.DB.Db) tr.DBChanges().Ignore() @@ -925,29 +939,34 @@ func TestAutomaticCommitmentTransfer(t *testing.T) { assert.HTTPRequest{ Method: http.MethodPost, Path: "/v1/domains/uuid-for-germany/projects/uuid-for-berlin/commitments/new", - Body: request(2), + Body: request(6), ExpectStatus: http.StatusCreated, }.Check(t, s.Handler) events := s.Auditor.RecordedEvents() assert.Equal(t, len(events), 2) - assert.Equal(t, events[0].Action, datamodel.ConsumeAction) + // first project: commitment creation POV + assert.Equal(t, events[0].Action, cadf.CreateAction) assert.Equal(t, len(events[0].Target.Attachments), 2) // changeRequest + transfer_status change - assert.Equal(t, events[0].Target.Attachments[1].Content, any(fmt.Sprintf(`{"%s":{"OldTransferStatus":"public","NewTransferStatus":""}}`, test.GenerateDummyCommitmentUUID(1)))) - assert.Equal(t, events[1].Action, cadf.CreateAction) - assert.Equal(t, len(events[1].Target.Attachments), 1) // changeRequest + assert.Equal(t, events[0].Target.Attachments[1].Content, any(fmt.Sprintf(`{"%s":{"OldTransferStatus":"public","NewTransferStatus":""},"%s":{"OldTransferStatus":"public","NewTransferStatus":""}}`, uuid1, uuid2))) + // second project: commitment consumption POV + assert.Equal(t, events[1].Action, datamodel.ConsumeAction) + assert.Equal(t, len(events[1].Target.Attachments), 2) // changeRequest + transfer_status change + assert.Equal(t, events[1].Target.Attachments[1].Content, any(fmt.Sprintf(`{"%s":{"OldTransferStatus":"public","NewTransferStatus":""},"%s":{"OldTransferStatus":"public","NewTransferStatus":""}}`, uuid1, uuid2))) tr.DBChanges().AssertEqualf(` DELETE FROM project_commitments WHERE id = 1 AND uuid = '%[1]s' AND transfer_token = 'dummyToken-1'; - INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, expires_at, superseded_at, creation_context_json, supersede_context_json) VALUES (1, '%[1]s', 2, 2, 'superseded', 1, '1 hour', %[3]d, 'dummy', 'dummy', %[4]d, %[3]d, '{}', '{"reason": "consume", "related_ids": [0], "related_uuids": ["%[2]s"]}'); - INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, confirmed_at, expires_at, creation_context_json) VALUES (2, '%[2]s', 1, 2, 'confirmed', 2, '2 hours', %[3]d, 'uuid-for-alice', 'alice@Default', %[3]d, %[5]d, '{"reason": "create"}'); - UPDATE services SET next_scrape_at = %[3]d WHERE id = 1 AND type = 'first' AND liquid_version = 1; - `, uuid, test.GenerateDummyCommitmentUUID(2), s.Clock.Now().Unix(), s.Clock.Now().Add(time.Hour).Unix(), s.Clock.Now().Add(2*time.Hour).Unix()) + INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, expires_at, superseded_at, creation_context_json, supersede_context_json) VALUES (1, '%[1]s', 2, 2, 'superseded', 3, '1 hour', %[4]d, 'dummy', 'dummy', %[5]d, %[4]d, '{}', '{"reason": "consume", "related_ids": [0], "related_uuids": ["%[3]s"]}'); + DELETE FROM project_commitments WHERE id = 2 AND uuid = '%[2]s' AND transfer_token = 'dummyToken-2'; + INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, confirmed_at, expires_at, superseded_at, creation_context_json, supersede_context_json) VALUES (2, '%[2]s', 2, 2, 'superseded', 3, '1 hour', %[4]d, 'dummy', 'dummy', %[4]d, %[5]d, %[4]d, '{}', '{"reason": "consume", "related_ids": [0], "related_uuids": ["%[3]s"]}'); + INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, confirmed_at, expires_at, creation_context_json) VALUES (3, '%[3]s', 1, 2, 'confirmed', 6, '2 hours', %[4]d, 'uuid-for-alice', 'alice@Default', %[4]d, %[6]d, '{"reason": "create"}'); + UPDATE services SET next_scrape_at = %[4]d WHERE id = 1 AND type = 'first' AND liquid_version = 1; + `, uuid1, uuid2, test.GenerateDummyCommitmentUUID(3), s.Clock.Now().Unix(), s.Clock.Now().Add(time.Hour).Unix(), s.Clock.Now().Add(2*time.Hour).Unix()) } func TestCommitmentDelegationToDB(t *testing.T) { s := setupCommitmentTest(t, testCommitmentsJSON) - // here, we modify the database so that the commitments for "first/capacity" go to the database for approval + // We modify the database so that the commitments for "first/capacity" go to the database for approval. s.MustDBExec(`UPDATE resources SET handles_commitments = FALSE;`) s.Clock.StepBy(10 * 24 * time.Hour) req := assert.JSONObject{ diff --git a/internal/audit/audit.go b/internal/audit/audit.go index 40823cd44..de904f7d8 100644 --- a/internal/audit/audit.go +++ b/internal/audit/audit.go @@ -18,9 +18,6 @@ import ( "github.com/sapcc/go-api-declarations/liquid" "github.com/sapcc/go-bits/audittools" "github.com/sapcc/go-bits/must" - - "github.com/sapcc/limes/internal/core" - "github.com/sapcc/limes/internal/db" ) // MaxQuotaEventTarget renders a cadf.Event.Target for a max_quota change event. @@ -124,18 +121,6 @@ func (t RateLimitEventTarget) Render() cadf.Resource { } } -// EnsureLiquidProjectMetadata guarantees that the given liquid.CommitmentChangeRequest -// contains project metadata for the given db.Project and db.Domain. -// The functions should be used before passing a liquid.CommitmentChangeRequest into an -// audit.CommitmentAttributeChangeset to be logged for auditing. For auditing purposes, -// the project metadata must be filled. It is important to call it for all involved projects. -func EnsureLiquidProjectMetadata(ccr liquid.CommitmentChangeRequest, project db.Project, domain db.Domain, serviceInfo liquid.ServiceInfo) liquid.CommitmentChangeRequest { - pcc := ccr.ByProject[project.UUID] - pcc.ProjectMetadata = Some(core.KeystoneProjectFromDB(project, core.KeystoneDomainFromDB(domain)).ForLiquid()) - ccr.ByProject[project.UUID] = pcc - return ccr -} - // redactLiquidProjectMetadataNames removes ProjectMedata of a // liquid.CommitmentChangeRequest. It is used to enable information-leak-free logging // of commitment changes where multiple projects are involved. @@ -150,8 +135,8 @@ func redactLiquidProjectMetadataNames(ccr liquid.CommitmentChangeRequest) liquid // CommitmentAttributeChangeset contains changes, which are not included in // liquid.CommitmentChangeRequest, but are relevant for auditing. type CommitmentAttributeChangeset struct { - OldTransferStatus Option[limesresources.CommitmentTransferStatus] // can be None, when the TransferStatus is stable - NewTransferStatus Option[limesresources.CommitmentTransferStatus] // can be None, when the TransferStatus is stable + OldTransferStatus limesresources.CommitmentTransferStatus + NewTransferStatus limesresources.CommitmentTransferStatus } // CommitmentEventTarget contains the structure for rendering a cadf.Event.Target for @@ -162,7 +147,7 @@ type CommitmentEventTarget struct { // must have at least one project, with one resource, with one commitment CommitmentChangeRequest liquid.CommitmentChangeRequest // can have one entry per commitment UUID - CommitmentAttributeChangeset map[liquid.CommitmentUUID]CommitmentAttributeChangeset + CommitmentAttributeChangesets map[liquid.CommitmentUUID]CommitmentAttributeChangeset } // ReplicateForAllProjects takes an audittools.Event and generates @@ -170,7 +155,7 @@ type CommitmentEventTarget struct { // the richCommitmentEventTarget for that project into the Target field. // It also redacts project and domain names from the CommitmentChangeRequest // to avoid information leaks in audit logs. -func (t CommitmentEventTarget) ReplicateForAllProjects(event audittools.Event) []audittools.Event { +func (t CommitmentEventTarget) ReplicateForAllProjects(event audittools.Event, overrideAction Option[cadf.Action], overrideProjectUUID Option[liquid.ProjectUUID]) []audittools.Event { // sort, to make audit event order deterministic projects := slices.Sorted(maps.Keys(t.CommitmentChangeRequest.ByProject)) var result []audittools.Event @@ -180,24 +165,32 @@ func (t CommitmentEventTarget) ReplicateForAllProjects(event audittools.Event) [ projectMetadataByProjectUUID[projectUUID] = pcc.ProjectMetadata } - for _, projectID := range projects { - projectMetadata := projectMetadataByProjectUUID[projectID] + for _, projectUUID := range projects { + projectMetadata := projectMetadataByProjectUUID[projectUUID] if pm, exists := projectMetadata.Unpack(); !exists { panic("attempted to create audit event target from CommitmentChangeRequest without ProjectMetadata") } else { + // With this logic we can achieve that multiple projects with transferred commitment(s) can keep + // the datamodel.ConsumeAction while the receiving project(s) can get cadf.CreateAction or datamodel.ConfirmAction. + newAction := event.Action + oAction, exists2 := overrideAction.Unpack() + oProjectUUID, exists3 := overrideProjectUUID.Unpack() + if exists2 && exists3 && oProjectUUID == projectUUID { + newAction = oAction + } result = append(result, audittools.Event{ Time: event.Time, Request: event.Request, User: event.User, ReasonCode: event.ReasonCode, - Action: event.Action, + Action: newAction, Target: richCommitmentEventTarget{ DomainID: pm.Domain.UUID, DomainName: pm.Domain.Name, ProjectID: liquid.ProjectUUID(pm.UUID), ProjectName: pm.Name, CommitmentChangeRequest: redactLiquidProjectMetadataNames(t.CommitmentChangeRequest), - CommitmentAttributeChangeset: t.CommitmentAttributeChangeset, + CommitmentAttributeChangeset: t.CommitmentAttributeChangesets, }, }) } diff --git a/internal/collector/capacity_scrape_test.go b/internal/collector/capacity_scrape_test.go index 5f78cd13e..9dacda05f 100644 --- a/internal/collector/capacity_scrape_test.go +++ b/internal/collector/capacity_scrape_test.go @@ -1391,7 +1391,7 @@ func Test_ScanCapacityWithCommitmentTakeover(t *testing.T) { %[4]s `, now.Unix(), uuid1, uuid2, timestampUpdates()) - // now we place a commitment that is in the same project, so it cannot be consume the transferable one; + // now we place a commitment that is in the same project, so it cannot consume the transferable one; // this checks that we avoid the loophole where the customer wants to get rid of an // old undeletable commitment by having it be consumed by a newer deletable one; // via API, this situation can only be achieved by first creating the planned commitment @@ -1793,6 +1793,82 @@ func Test_ScanCapacityWithCommitmentTakeover(t *testing.T) { UPDATE project_commitments SET status = 'confirmed', confirmed_at = %[1]d WHERE id = 22 AND uuid = '%[9]s' AND transfer_token = NULL; %[10]s `, now.Unix(), creation3.Unix(), transferStartedAt.Unix(), confirmBy.Unix(), confirmation.Unix(), expiry.Unix(), uuid16, uuid19, uuid22, timestampUpdates()) + + // Now, we want to check that transfers of confirmed commitments free capacity up. + // Additionally, this should enable multiple confirmations in a row, which would otherwise not have enough capacity. + // The capacity of firstCapacityAZOne is 42, committed are currently 22. + // The first transferable commitment gets confirmed immediately, making committed 41. + // The second transferable commitment gets consumed while still in planned state. + creation = s.Clock.Now() + expiry = s.Clock.Now().Add(10 * oneDay) + + uuid23 := add(db.ProjectCommitment{ + UUID: s.Collector.GenerateProjectCommitmentUUID(), + ProjectID: berlin, + AZResourceID: firstCapacityAZOne, + Amount: 19, + CreatedAt: creation, + Duration: committedForTenDays, + TransferToken: Some(s.Collector.GenerateTransferToken()), + TransferStatus: limesresources.CommitmentTransferStatusPublic, + TransferStartedAt: Some(creation), + }) + s.Clock.StepBy(1 * time.Hour) + must.SucceedT(t, jobloop.ProcessMany(job, s.Ctx, len(s.Cluster.LiquidConnections))) + + confirmation = s.Clock.Now().Add(-5 * time.Second) + creation2 = s.Clock.Now() + expiry2 = s.Clock.Now().Add(10 * oneDay) + uuid24 := add(db.ProjectCommitment{ + UUID: s.Collector.GenerateProjectCommitmentUUID(), + ProjectID: paris, + AZResourceID: firstCapacityAZOne, + Amount: 4, + CreatedAt: s.Clock.Now(), + Duration: committedForTenDays, + }) + uuid25 := add(db.ProjectCommitment{ + UUID: s.Collector.GenerateProjectCommitmentUUID(), + ProjectID: paris, + AZResourceID: firstCapacityAZOne, + Amount: 16, + CreatedAt: s.Clock.Now(), + Duration: committedForTenDays, + }) + uuid26 := add(db.ProjectCommitment{ + UUID: s.Collector.GenerateProjectCommitmentUUID(), + ProjectID: berlin, + AZResourceID: firstCapacityAZOne, + Amount: 10, + CreatedAt: creation2, + Duration: committedForTenDays, + TransferToken: Some(s.Collector.GenerateTransferToken()), + TransferStatus: limesresources.CommitmentTransferStatusPublic, + TransferStartedAt: Some(creation2), + }) + tr.DBChanges().Ignore() + + // We expect that the first commitment gets transferred completely in 2 steps, freeing 19 capacity. + // The second transferable commitment is still planned and 1 capacity is transferred, the rest is a leftover. + s.Clock.StepBy(1 * time.Hour) + must.SucceedT(t, jobloop.ProcessMany(job, s.Ctx, len(s.Cluster.LiquidConnections))) + + now = s.Clock.Now().Add(-5 * time.Second) + tr.DBChanges().AssertEqualf(` + UPDATE project_az_resources SET quota = 2 WHERE id = 2 AND project_id = 1 AND az_resource_id = 2; + UPDATE project_az_resources SET quota = 24 WHERE id = 30 AND project_id = 3 AND az_resource_id = 2; + UPDATE project_az_resources SET quota = 24 WHERE id = 32 AND project_id = 3 AND az_resource_id = 4; + UPDATE project_az_resources SET quota = 2 WHERE id = 4 AND project_id = 1 AND az_resource_id = 4; + DELETE FROM project_commitments WHERE id = 23 AND uuid = '%[7]s' AND transfer_token = 'dummyToken-11'; + INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, confirmed_at, expires_at, superseded_at, creation_context_json, supersede_context_json) VALUES (23, '%[7]s', 1, 2, 'superseded', 19, '10 days', %[2]d, 'dummy', 'dummy', %[6]d, %[4]d, %[1]d, '{}', '{"reason": "consume", "related_ids": [24], "related_uuids": ["%[8]s"]}'); + UPDATE project_commitments SET status = 'confirmed', confirmed_at = %[1]d WHERE id = 24 AND uuid = '%[8]s' AND transfer_token = NULL; + UPDATE project_commitments SET status = 'confirmed', confirmed_at = %[1]d WHERE id = 25 AND uuid = '%[9]s' AND transfer_token = NULL; + DELETE FROM project_commitments WHERE id = 26 AND uuid = '%[10]s' AND transfer_token = 'dummyToken-12'; + INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, expires_at, superseded_at, creation_context_json, supersede_context_json) VALUES (26, '%[10]s', 1, 2, 'superseded', 10, '10 days', %[3]d, 'dummy', 'dummy', %[5]d, %[1]d, '{}', '{"reason": "consume", "related_ids": [25], "related_uuids": ["%[9]s"]}'); + INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, confirmed_at, expires_at, superseded_at, creation_context_json, supersede_context_json) VALUES (27, '%[11]s', 1, 2, 'superseded', 15, '10 days', %[1]d, 'dummy', 'dummy', %[6]d, %[4]d, %[1]d, '{"reason": "split", "related_ids": [23], "related_uuids": ["%[7]s"]}', '{"reason": "consume", "related_ids": [25], "related_uuids": ["b7a56873-cd77-4f2c-446d-369b649430b6"]}'); + INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, expires_at, transfer_status, transfer_token, creation_context_json, transfer_started_at) VALUES (28, '%[12]s', 1, 2, 'pending', 9, '10 days', %[1]d, 'dummy', 'dummy', %[5]d, 'public', 'dummyToken-14', '{"reason": "split", "related_ids": [26], "related_uuids": ["%[10]s"]}', %[3]d); + %[13]s + `, now.Unix(), creation.Unix(), creation2.Unix(), expiry.Unix(), expiry2.Unix(), confirmation.Unix(), uuid23, uuid24, uuid25, uuid26, test.GenerateDummyCommitmentUUID(27), test.GenerateDummyCommitmentUUID(28), timestampUpdates()) } func TestScanCapacityWithCommitmentsChecksLiquidForCapacity(t *testing.T) { @@ -2107,7 +2183,7 @@ func TestScanCapacityWithMailNotification(t *testing.T) { events = s.Auditor.RecordedEvents() assert.Equal(t, len(events), 2) // one confirmation, one transfer assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "confirm" })), 1) - assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "confirm" })[0].Target.Attachments), 1) // no changes to the transfer status, just 1 entry + assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "confirm" })[0].Target.Attachments), 2) // transfer_status changes assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "consume" })), 1) assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "consume" })[0].Target.Attachments), 2) // transfer_status changes @@ -2155,7 +2231,7 @@ func TestScanCapacityWithMailNotification(t *testing.T) { events = s.Auditor.RecordedEvents() assert.Equal(t, len(events), 2) // one confirmation, one transfer assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "confirm" })), 1) - assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "confirm" })[0].Target.Attachments), 1) // no changes to the transfer status, just 1 entry + assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "confirm" })[0].Target.Attachments), 2) // / transfer_status changes assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "consume" })), 1) assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "consume" })[0].Target.Attachments), 2) // transfer_status changes @@ -2191,13 +2267,13 @@ func TestScanCapacityWithMailNotification(t *testing.T) { INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, confirmed_at, expires_at, transfer_status, transfer_token, creation_context_json, transfer_started_at) VALUES (15, '%[9]s', 2, 9, 'confirmed', 6, '10 days', 475260, 'dummy', 'dummy', 388850, 1166440, 'public', 'dummyToken-6', '{"reason": "split", "related_ids": [14], "related_uuids": ["%[8]s"]}', 302440); DELETE FROM project_commitments WHERE id = 9 AND uuid = '%[3]s' AND transfer_token = 'dummyToken-3'; INSERT INTO project_commitments (id, uuid, project_id, az_resource_id, status, amount, duration, created_at, creator_uuid, creator_name, confirmed_at, expires_at, superseded_at, creation_context_json, supersede_context_json) VALUES (9, '%[3]s', 2, 9, 'superseded', 9, '10 days', 388850, 'dummy', 'dummy', 388850, 1166440, 475260, '{"reason": "split", "related_ids": [7], "related_uuids": ["7902699b-e42c-4a8e-46fb-bb4501726517"]}', '{"reason": "consume", "related_ids": [10], "related_uuids": ["%[4]s"]}'); - INSERT INTO project_mail_notifications (id, project_id, subject, body, next_submission_at) VALUES (8, 2, 'Your recent commitment transfers', 'Domain:germany Project:dresden Creator:dummy Amount:9 Duration:10 days Date:1970-01-05 Service:service Resource:resource AZ:az-one Leftover:6', %[1]d); + INSERT INTO project_mail_notifications (id, project_id, subject, body, next_submission_at) VALUES (8, 2, 'Your recent commitment transfers', 'Domain:germany Project:dresden Creator:dummy Amount:9 Duration:10 days Date:1970-01-06 Service:service Resource:resource AZ:az-one Leftover:6', %[1]d); %[10]s `, scrapedAt2.Unix(), uuid7, uuid9, resultUUIDs[0], resultUUIDs[1], resultUUIDs[2], test.GenerateDummyCommitmentUUID(13), test.GenerateDummyCommitmentUUID(14), test.GenerateDummyCommitmentUUID(15), timestampUpdates()) events = s.Auditor.RecordedEvents() assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "confirm" })), 3) - assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "confirm" })[0].Target.Attachments), 1) // no changes to the transfer status, just 1 entry - assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "consume" })), 1) + assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "confirm" })[0].Target.Attachments), 2) // transfer_status changes + assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "consume" })), 3) assert.Equal(t, len(filterSlice(events, func(e cadf.Event) bool { return e.Action == "consume" })[0].Target.Attachments), 2) // transfer_status changes } diff --git a/internal/datamodel/commitment.go b/internal/datamodel/commitment.go index d930c0986..e53629b1d 100644 --- a/internal/datamodel/commitment.go +++ b/internal/datamodel/commitment.go @@ -223,11 +223,10 @@ func DelegateChangeCommitments(ctx context.Context, cluster *core.Cluster, req l return result, nil } -// LiquidProjectMetadataFromDBProject converts a db.Project into liquid.ProjectMetadata -// only if the given serviceInfo requires it for commitment handling. -func LiquidProjectMetadataFromDBProject(dbProject db.Project, domain db.Domain, serviceInfo liquid.ServiceInfo) Option[liquid.ProjectMetadata] { - if !serviceInfo.CommitmentHandlingNeedsProjectMetadata { - return None[liquid.ProjectMetadata]() - } +// LiquidProjectMetadataFromDBProject converts a db.Project into liquid.ProjectMetadata. +// We use this function regardless of `liquid.ServiceInfo.CommitmentHandlingNeedsProjectMetadata`, so that +// ProjectMetadata is always filled for Limes-internal use. Before delegating to the liquid, we might remove +// it again if not needed. +func LiquidProjectMetadataFromDBProject(dbProject db.Project, domain db.Domain) Option[liquid.ProjectMetadata] { return Some(core.KeystoneProjectFromDB(dbProject, core.KeystoneDomain{UUID: domain.UUID, Name: domain.Name}).ForLiquid()) } diff --git a/internal/datamodel/confirm_project_commitments.go b/internal/datamodel/confirm_project_commitments.go index b6e2ba473..4cdaa4847 100644 --- a/internal/datamodel/confirm_project_commitments.go +++ b/internal/datamodel/confirm_project_commitments.go @@ -7,7 +7,6 @@ import ( "context" "fmt" "maps" - "net/http" "slices" "time" @@ -121,30 +120,30 @@ func CanAcceptCommitmentChangeRequest(req liquid.CommitmentChangeRequest, loc co // them as possible given the currently available capacity. Simultaneously, it // releases transferable commitments that can be used to satisfy the pending ones. func ConfirmPendingCommitments(ctx context.Context, loc core.AZResourceLocation, cluster *core.Cluster, dbi db.Interface, now time.Time, generateProjectCommitmentUUID func() liquid.CommitmentUUID, generateTransferToken func() string, auditContext audit.Context) (auditEvents []audittools.Event, err error) { - // load service info (used to generate liquid.ProjectMetadata) - maybeServiceInfo, err := cluster.InfoForService(loc.ServiceType) - if err != nil { - return nil, err - } - serviceInfo, ok := maybeServiceInfo.Unpack() - if !ok { - return nil, fmt.Errorf("serviceInfo not found when trying to confirm commitments for %s", loc.ServiceType) - } - // load confirmable commitments var confirmableCommitments []db.ProjectCommitment - confirmedCommitmentIDs := make(map[db.ProjectID][]db.ProjectCommitmentID) + confirmedCommitmentsByProjectID := make(map[db.ProjectID][]*db.ProjectCommitment) queryArgs := []any{loc.ServiceType, loc.ResourceName, loc.AvailabilityZone} _, err = dbi.Select(&confirmableCommitments, getConfirmableCommitmentsQuery, queryArgs...) if err != nil { return nil, fmt.Errorf("while enumerating confirmable commitments for %s: %w", loc.ScopeString(), err) } - // optimization: do not load allocation stats if we do not have anything to confirm + // optimization: do not do more loading, if we do not have anything to confirm if len(confirmableCommitments) == 0 { return nil, nil } + // load service info (used to generate liquid.ProjectMetadata) + maybeServiceInfo, err := cluster.InfoForService(loc.ServiceType) + if err != nil { + return nil, err + } + serviceInfo, ok := maybeServiceInfo.Unpack() + if !ok { + return nil, fmt.Errorf("serviceInfo not found when trying to confirm commitments for %s", loc.ServiceType) + } + // load affected projects and domains affectedProjectIDs := make(map[db.ProjectID]struct{}) for _, c := range confirmableCommitments { @@ -168,137 +167,98 @@ func ConfirmPendingCommitments(ctx context.Context, loc core.AZResourceLocation, } // initiate cache of transferable commitments - transferableCommitmentCache, err := NewTransferableCommitmentCache(dbi, serviceInfo, loc, now, generateProjectCommitmentUUID, generateTransferToken, transferTemplate) - if err != nil { - return nil, err - } - - // load allocation stats - statsByAZ, err := collectAZAllocationStats(loc.ServiceType, loc.ResourceName, Some(loc.AvailabilityZone), cluster, dbi) + transferableCommitmentCache, err := NewTransferableCommitmentCache(dbi, cluster, serviceInfo, loc, now, generateProjectCommitmentUUID, generateTransferToken, transferTemplate) if err != nil { return nil, err } - stats := statsByAZ[loc.AvailabilityZone] - - // initialize map to hold CCRs for audit events - ccrs := make(map[liquid.CommitmentUUID]liquid.CommitmentChangeRequest, 0) // foreach confirmable commitment in the order to be confirmed for _, cc := range confirmableCommitments { - // ignore commitments that do not fit - logg.Debug("checking ConfirmPendingCommitments in %s: commitmentID = %d, projectID = %d, amount = %d", - loc.ShortScopeString(), cc.ID, cc.ProjectID, cc.Amount) - project := affectedProjectsByID[cc.ProjectID] - domain := affectedDomainsByID[project.DomainID] - - var capacityAccepted bool - capacityAccepted, ccr, err := delegateChangeCommitmentsWithShortcut(ctx, cluster, dbi, loc, serviceInfo, project, domain, cc, stats) - if err != nil { - return nil, fmt.Errorf("while checking acceptance of commitment ID=%d for %s: %w", cc.ID, loc.ScopeString(), err) - } - if !capacityAccepted { - continue - } - ccrs[cc.UUID] = ccr - // if a commitment was transferred in this iteration already, we do not need to confirm it - // if partially transferred, the leftover commitment is added to the transferable commitments and considered separately + // If a commitment was transferred in this iteration already, we do not need to confirm it. + // If partially transferred, the leftover is added to the transferable commitments instead of the superseded one. if transferableCommitmentCache.CommitmentWasTransferred(cc.ID, cc.ProjectID) { continue } - err = transferableCommitmentCache.CheckAndConsume(cc, stats.ProjectStats[cc.ProjectID].Committed) + // First, we check whether we can consume transferable commitments and have the necessary capacity. + project := affectedProjectsByID[cc.ProjectID] + domain := affectedDomainsByID[project.DomainID] + result, err := transferableCommitmentCache.CanConfirmWithTransfers(ctx, cc, project, domain, false, false, auditContext, ConfirmAction) if err != nil { return nil, err } - // confirm the commitment - _, err = dbi.Exec(`UPDATE project_commitments SET confirmed_at = $1, status = $2 WHERE id = $3`, - now, liquid.CommitmentStatusConfirmed, cc.ID) + // When we cannot confirm the commitment, we check with the next one. This can lead to + // smaller but later created commitments to be confirmed earlier, but that is acceptable. + if result.RejectionReason != "" { + continue + } + + // capacity is sufficient --> confirm the commitment + cc.ConfirmedAt = Some(now) + cc.Status = liquid.CommitmentStatusConfirmed + _, err = dbi.Update(&cc) if err != nil { return nil, fmt.Errorf("while confirming commitment ID=%d for %s: %w", cc.ID, loc.ScopeString(), err) } transferableCommitmentCache.ConfirmTransferableCommitmentIfExists(cc.ID, now) - confirmedCommitmentIDs[cc.ProjectID] = append(confirmedCommitmentIDs[cc.ProjectID], cc.ID) - - // block its allocation from being committed again in this loop - oldStats := stats.ProjectStats[cc.ProjectID] - stats.ProjectStats[cc.ProjectID] = projectAZAllocationStats{ - Committed: oldStats.Committed + cc.Amount, - Usage: oldStats.Usage, - } + confirmedCommitmentsByProjectID[cc.ProjectID] = append(confirmedCommitmentsByProjectID[cc.ProjectID], &cc) } - // gather some prerequisites for the mail notifications + // generate mail notifications for commitment transfers apiIdentity := cluster.BehaviorForResource(loc.ServiceType, loc.ResourceName).IdentityInV1API - - // generate audit events and mail notifications for commitment transfers - ae, err := transferableCommitmentCache.GenerateAuditEventsAndMails(apiIdentity, auditContext) + err = transferableCommitmentCache.GenerateTransferMails(apiIdentity) if err != nil { return nil, err } - auditEvents = append(auditEvents, ae...) - for _, projectID := range slices.Sorted(maps.Keys(confirmedCommitmentIDs)) { - confirmations := confirmedCommitmentIDs[projectID] - // for commitments which get confirmed first and then transferred, we remove the mail for confirmation - // to avoid duplicate notification mails + // retrieve audit events for commitment transfers + auditEvents = append(auditEvents, transferableCommitmentCache.RetrieveAuditEvents()...) + + // generate mail notifications for commitment confirmations + for _, projectID := range slices.Sorted(maps.Keys(confirmedCommitmentsByProjectID)) { + confirmedCommitments := confirmedCommitmentsByProjectID[projectID] + // For commitments which get confirmed first and then transferred, we remove the mail for confirmation + // to avoid duplicate notification mails. notificationsForProject := transferableCommitmentCache.getTransferredCommitmentsForProject(projectID) for cID := range notificationsForProject { - confirmations = slices.DeleteFunc(confirmations, func(id db.ProjectCommitmentID) bool { return id == cID }) + confirmedCommitments = slices.DeleteFunc(confirmedCommitments, func(c *db.ProjectCommitment) bool { return c.ID == cID }) } - // generate audit events and mail notifications for commitment confirmations - ae, err = generateAuditEventsAndMails(confirmationTemplate, dbi, loc, apiIdentity, projectID, auditContext, confirmations, ccrs, now) + affectedProject := affectedProjectsByID[projectID] + affectedDomain := affectedDomainsByID[affectedProject.DomainID] + err = generateConfirmationMails(confirmationTemplate, dbi, loc, apiIdentity, affectedProject, affectedDomain, confirmedCommitments, now) if err != nil { return nil, err } - auditEvents = append(auditEvents, ae...) } return auditEvents, nil } -func generateAuditEventsAndMails(mailTemplate Option[core.MailTemplate], dbi db.Interface, loc core.AZResourceLocation, apiIdentity core.ResourceRef, projectID db.ProjectID, auditContext audit.Context, - confirmedCommitmentIDs []db.ProjectCommitmentID, ccrs map[liquid.CommitmentUUID]liquid.CommitmentChangeRequest, now time.Time) (auditEvents []audittools.Event, err error) { - - var ( - n core.CommitmentGroupNotification - domainUUID string - projectUUID liquid.ProjectUUID - ) - err = dbi.QueryRow("SELECT d.uuid, d.name, p.uuid, p.name FROM domains d JOIN projects p ON d.id = p.domain_id where p.id = $1", projectID).Scan(&domainUUID, &n.DomainName, &projectUUID, &n.ProjectName) - if err != nil { - return auditEvents, err +func generateConfirmationMails(mailTemplate Option[core.MailTemplate], dbi db.Interface, loc core.AZResourceLocation, apiIdentity core.ResourceRef, project db.Project, domain db.Domain, confirmedCommitments []*db.ProjectCommitment, now time.Time) error { + // The system can be configured to not send mails (e.g. for test systems). + tpl, tplExists := mailTemplate.Unpack() + if !tplExists { + return nil } - commitmentsByID, err := db.BuildIndexOfDBResult(dbi, func(c db.ProjectCommitment) db.ProjectCommitmentID { return c.ID }, `SELECT * FROM project_commitments WHERE id = ANY($1)`, pq.Array(confirmedCommitmentIDs)) - if err != nil { - return auditEvents, err + n := core.CommitmentGroupNotification{ + DomainName: domain.Name, + ProjectName: project.Name, + Commitments: make([]core.CommitmentNotification, 0, len(confirmedCommitments)), } - for _, cID := range confirmedCommitmentIDs { - c, exists := commitmentsByID[cID] - if !exists { - return auditEvents, fmt.Errorf("tried to generate mail notification for non-existent commitment ID %d", cID) - } - confirmedAt := c.ConfirmedAt.UnwrapOr(time.Unix(0, 0)) // the UnwrapOr() is defense in depth, it should never be relevant because we only notify for confirmed commitments here - - // push one confirmation audit event per commitment, because they belong to separate CCRs - auditEvents = append(auditEvents, audit.CommitmentEventTarget{ - CommitmentChangeRequest: ccrs[c.UUID], - }.ReplicateForAllProjects(audittools.Event{ - Time: now, - Request: auditContext.Request, - User: auditContext.UserIdentity, - ReasonCode: http.StatusOK, - Action: ConfirmAction, - })...) - + for _, c := range confirmedCommitments { + // The user can choose to not be notified on confirmation. if !c.NotifyOnConfirm { continue } + + // also defense in depth: we only generate mails for confirmed commitments + confirmedAt := c.ConfirmedAt.UnwrapOr(time.Unix(0, 0)) // the UnwrapOr() is defense in depth, it should never be relevant because we only notify for confirmed commitments here n.Commitments = append(n.Commitments, core.CommitmentNotification{ - Commitment: c, + Commitment: *c, DateString: confirmedAt.Format(time.DateOnly), // TODO: we actually don't want to have api-named props in AZResourceLocation. Replace the template and the code simultaneously. Resource: core.AZResourceLocation{ @@ -308,68 +268,16 @@ func generateAuditEventsAndMails(mailTemplate Option[core.MailTemplate], dbi db. }, }) } - if tpl, exists := mailTemplate.Unpack(); len(n.Commitments) != 0 && exists { + if len(n.Commitments) != 0 { // push mail notifications - mail, err := tpl.Render(n, projectID, now) + mail, err := tpl.Render(n, project.ID, now) if err != nil { - return auditEvents, err + return err } err = dbi.Insert(&mail) if err != nil { - return auditEvents, err + return err } } - return auditEvents, nil -} - -func delegateChangeCommitmentsWithShortcut(ctx context.Context, cluster *core.Cluster, dbi db.Interface, loc core.AZResourceLocation, serviceInfo liquid.ServiceInfo, project db.Project, domain db.Domain, commitment db.ProjectCommitment, stats clusterAZAllocationStats) (accepted bool, ccr liquid.CommitmentChangeRequest, err error) { - ccr = liquid.CommitmentChangeRequest{ - AZ: loc.AvailabilityZone, - InfoVersion: serviceInfo.Version, - ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ - project.UUID: { - ProjectMetadata: LiquidProjectMetadataFromDBProject(project, domain, serviceInfo), - ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ - loc.ResourceName: { - TotalConfirmedBefore: stats.ProjectStats[commitment.ProjectID].Committed, - TotalConfirmedAfter: stats.ProjectStats[commitment.ProjectID].Committed + commitment.Amount, - // TODO: change when introducing "guaranteed" commitments - TotalGuaranteedBefore: 0, - TotalGuaranteedAfter: 0, - Commitments: []liquid.Commitment{ - { - UUID: commitment.UUID, - OldStatus: Some(commitment.Status), - NewStatus: Some(liquid.CommitmentStatusConfirmed), - Amount: commitment.Amount, - ConfirmBy: commitment.ConfirmBy, - ExpiresAt: commitment.ExpiresAt, - }, - }, - }, - }, - }, - }, - } - - // optimization: we check locally, when we know that the resource does not manage commitments - // this avoids having to re-load the stats later in the callchain. - if !serviceInfo.Resources[loc.ResourceName].HandlesCommitments { - additions := map[db.ProjectID]uint64{commitment.ProjectID: commitment.Amount} - behavior := cluster.CommitmentBehaviorForResource(loc.ServiceType, loc.ResourceName) - accepted = stats.CanAcceptCommitmentChanges(additions, nil, behavior) - } else { - commitmentChangeResponse, err := DelegateChangeCommitments(ctx, cluster, ccr, loc, serviceInfo, dbi) - if err != nil { - return false, liquid.CommitmentChangeRequest{}, err - } - - accepted = commitmentChangeResponse.RejectionReason == "" - if !accepted { - logg.Info("commitment not accepted for %s: %s", loc.ShortScopeString(), commitmentChangeResponse.RejectionReason) - } - } - - // as the totalConfirmed will increase, we don't need to check for RequiresConfirmation() here - return accepted, audit.EnsureLiquidProjectMetadata(ccr, project, domain, serviceInfo), nil + return nil } diff --git a/internal/datamodel/consume_transferable_commitments.go b/internal/datamodel/consume_transferable_commitments.go index 5ce1dbfe7..3c945b06b 100644 --- a/internal/datamodel/consume_transferable_commitments.go +++ b/internal/datamodel/consume_transferable_commitments.go @@ -4,6 +4,7 @@ package datamodel import ( + "context" "encoding/json" "fmt" "maps" @@ -13,9 +14,11 @@ import ( "github.com/lib/pq" . "github.com/majewsky/gg/option" + "github.com/sapcc/go-api-declarations/cadf" limesresources "github.com/sapcc/go-api-declarations/limes/resources" "github.com/sapcc/go-api-declarations/liquid" "github.com/sapcc/go-bits/audittools" + "github.com/sapcc/go-bits/logg" "github.com/sapcc/go-bits/sqlext" "github.com/sapcc/limes/internal/audit" @@ -44,23 +47,30 @@ var getTransferableCommitmentsQuery = sqlext.SimplifyWhitespace(db.ExpandEnumPla // TransferableCommitmentCache handles the consumption of transferable commitments. // It's main functionality is to cache the state of the these commitments between -// calls of CheckAndConsume, so that continuous reloading is avoided. Therefore, an +// calls of CanConfirmWithTransfers, so that continuous reloading is avoided. Therefore, an // instance of it should be obtained by NewTransferableCommitmentManager and reused // between subsequent calls. type TransferableCommitmentCache struct { // transferableCommitments holds the order of the transferableCommitments to consume in (see query). - transferableCommitments []db.ProjectCommitment + transferableCommitments []*db.ProjectCommitment // transferableCommitmentsByID allows quick lookup of commitments by their ID (see ReplaceTransferableCommitment). transferableCommitmentsByID map[db.ProjectCommitmentID]*db.ProjectCommitment // transferredCommitmentIDs holds the IDs of commitments that have already been transferred. transferredCommitmentIDs map[db.ProjectID]map[db.ProjectCommitmentID]commitmentTransferLeftover - // affectedProjectsByID hold all projects that have transferable commitments - affectedProjectsByID map[db.ProjectID]db.Project - // affectedDomainsByID hold all domains that have projects with transferable commitments - affectedDomainsByID map[db.DomainID]db.Domain + + // project and domain structures for use in requests, mails etc. + // NOTE: These get enriched with data of projects/ domains that have confirmable commitments in the process. + affectedProjectsByID map[db.ProjectID]db.Project + affectedProjectsByUUID map[liquid.ProjectUUID]db.Project + affectedDomainsByID map[db.DomainID]db.Domain + + // in order to deduplicate audit events between confirmation outside the cache and transfer inside, + // we keep track of the audit events by confirmed commitment. At the end, RetrieveAuditEvents should be called. + auditEventsByConfirmedCommitmentUUID map[liquid.CommitmentUUID][]audittools.Event // utilities dbi db.Interface + cluster *core.Cluster serviceInfo liquid.ServiceInfo loc core.AZResourceLocation now time.Time @@ -68,13 +78,13 @@ type TransferableCommitmentCache struct { generateTransferToken func() string mailTemplate Option[core.MailTemplate] - // the following fields are used for caching between CheckAndConsume() and GenerateAuditEventsAndMails() - ccrs map[liquid.CommitmentUUID]liquid.CommitmentChangeRequest - cacs map[liquid.CommitmentUUID]audit.CommitmentAttributeChangeset + // we have to keep track of stats always, to provide totalConfirmed in CCRs + liquidHandlesCommitments bool + stats clusterAZAllocationStats } // NewTransferableCommitmentCache builds a TransferableCommitmentCache and fills it. -func NewTransferableCommitmentCache(dbi db.Interface, serviceInfo liquid.ServiceInfo, loc core.AZResourceLocation, now time.Time, generateProjectCommitmentUUID func() liquid.CommitmentUUID, generateTransferToken func() string, mailTemplate Option[core.MailTemplate]) (t TransferableCommitmentCache, err error) { +func NewTransferableCommitmentCache(dbi db.Interface, cluster *core.Cluster, serviceInfo liquid.ServiceInfo, loc core.AZResourceLocation, now time.Time, generateProjectCommitmentUUID func() liquid.CommitmentUUID, generateTransferToken func() string, mailTemplate Option[core.MailTemplate]) (t TransferableCommitmentCache, err error) { queryArgs := []any{loc.ServiceType, loc.ResourceName, loc.AvailabilityZone} _, err = dbi.Select(&t.transferableCommitments, getTransferableCommitmentsQuery, queryArgs...) if err != nil { @@ -83,23 +93,31 @@ func NewTransferableCommitmentCache(dbi db.Interface, serviceInfo liquid.Service t.transferableCommitmentsByID = make(map[db.ProjectCommitmentID]*db.ProjectCommitment, len(t.transferableCommitments)) affectedProjectIDs := make(map[db.ProjectID]struct{}) for i := range t.transferableCommitments { - t.transferableCommitmentsByID[t.transferableCommitments[i].ID] = &t.transferableCommitments[i] + t.transferableCommitmentsByID[t.transferableCommitments[i].ID] = t.transferableCommitments[i] affectedProjectIDs[t.transferableCommitments[i].ProjectID] = struct{}{} } - t.transferredCommitmentIDs = make(map[db.ProjectID]map[db.ProjectCommitmentID]commitmentTransferLeftover) + // project and domain structures t.affectedProjectsByID, err = db.BuildIndexOfDBResult(dbi, func(p db.Project) db.ProjectID { return p.ID }, `SELECT * from projects WHERE id = ANY($1)`, pq.Array(slices.Collect(maps.Keys(affectedProjectIDs)))) if err != nil { return t, fmt.Errorf("while loading projects with transferable commitments for %s: %w", loc.ScopeString(), err) } + t.affectedProjectsByUUID = make(map[liquid.ProjectUUID]db.Project) + for _, project := range t.affectedProjectsByID { + t.affectedProjectsByUUID[project.UUID] = project + } t.affectedDomainsByID, err = db.BuildIndexOfDBResult(dbi, func(d db.Domain) db.DomainID { return d.ID }, `SELECT * from domains WHERE id IN (SELECT domain_id FROM projects WHERE id = ANY($1))`, pq.Array(slices.Collect(maps.Keys(affectedProjectIDs)))) if err != nil { return t, fmt.Errorf("while loading domains with projects with transferable commitments for %s: %w", loc.ScopeString(), err) } + // prep audit event storage + t.auditEventsByConfirmedCommitmentUUID = make(map[liquid.CommitmentUUID][]audittools.Event) + // fill utilities t.dbi = dbi + t.cluster = cluster t.serviceInfo = serviceInfo t.loc = loc t.now = now @@ -107,38 +125,89 @@ func NewTransferableCommitmentCache(dbi db.Interface, serviceInfo liquid.Service t.generateTransferToken = generateTransferToken t.mailTemplate = mailTemplate - // initialize caching fields - t.ccrs = make(map[liquid.CommitmentUUID]liquid.CommitmentChangeRequest) - t.cacs = make(map[liquid.CommitmentUUID]audit.CommitmentAttributeChangeset) + // determine whether liquid handles commitments for this resource + t.liquidHandlesCommitments = t.serviceInfo.Resources[t.loc.ResourceName].HandlesCommitments + statsByAZ, err := collectAZAllocationStats(loc.ServiceType, loc.ResourceName, Some(loc.AvailabilityZone), cluster, dbi) + if err != nil { + return t, fmt.Errorf("while collecting AZ stats for %s: %w", loc.ScopeString(), err) + } + t.stats = statsByAZ[loc.AvailabilityZone] return t, nil } -// CheckAndConsume checks whether the given db.ProjectCommitment can take over 1 to n -// of the transferableCommitments. If so, the transferableCommitments get modified -// according to the new state. The commitment to takeover the transferred amount is -// not subject to any change in this operation. All relations to the consuming commitment -// are stored in the SupersedeContextJSON of the consumed transferableCommitments. +// CanConfirmWithTransfers checks whether the given db.ProjectCommitment can take over 1 to n +// of the transferableCommitments and whether the missing amount can be confirmed. +// If so, the transferableCommitments get modified according to the new state. The commitment +// to takeover the transferred amount is not subject to change in this operation, i.e. any +// update to it should be done after calling this function and any errors should cancel the transaction. +// All relations to the consuming commitment are stored in the SupersedeContextJSON of the consumed +// transferableCommitments. // // Commitment consumption follows the following rules: // We consume maximum the amount of the given commitment in one run. // The status of a transferableCommitment to be consumed does not matter. +// When the missing amount does not fit into the capacity, no commitments are consumed at all. // When a commitment is both pending and transferable, the handling depends on the order: // When confirmed first, it might be taken over later anyway. Therefore, when confirming a -// commitment while using the cache, always call ConfirmTransferableCommitmentIfExists -// when a commitment gets confirmed outside of the cache. -// When a commitment gets transferred first, it should not get confirmed later. For this, -// CommitmentWasTransferred should be used to check before confirming a commitment. -// All transfers will lead to a mail which contains the leftover amount, so that the customer -// can track the whole processing of the transferred commitment over time. -func (t *TransferableCommitmentCache) CheckAndConsume(c db.ProjectCommitment, currentTotalConfirmed uint64) (err error) { - overallTransferredAmount := uint64(0) +// commitment outside the cache, always call ConfirmTransferableCommitmentIfExists. When a +// commitment gets transferred first, it should not get confirmed later. For this, +// CommitmentWasTransferred should be used to check before confirming a commitment outside the cache. +// To enable mail-collation, GenerateTransferMails should be called after all calls to CanConfirmWithTransfers. +func (t *TransferableCommitmentCache) CanConfirmWithTransfers(ctx context.Context, c db.ProjectCommitment, project db.Project, domain db.Domain, isNew, dryRun bool, auditContext audit.Context, auditAction cadf.Action) (result liquid.CommitmentChangeResponse, err error) { + // We add the project and domain to the affected lists, so that we can refer to them in private functions more easily. + t.affectedProjectsByID[project.ID] = project + t.affectedProjectsByUUID[project.UUID] = project + t.affectedDomainsByID[domain.ID] = domain + + // For checking the capacity we place the new commitment into a CCR, that we will enrich with the transfers below. + oldStatus := Some(c.Status) + if isNew { + oldStatus = None[liquid.CommitmentStatus]() + } + ccr := liquid.CommitmentChangeRequest{ + DryRun: dryRun, + AZ: t.loc.AvailabilityZone, + InfoVersion: t.serviceInfo.Version, + ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ + project.UUID: { + ProjectMetadata: LiquidProjectMetadataFromDBProject(project, domain), + ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ + t.loc.ResourceName: { + TotalConfirmedBefore: t.stats.ProjectStats[c.ProjectID].Committed, + TotalConfirmedAfter: t.stats.ProjectStats[c.ProjectID].Committed + c.Amount, + // TODO: change when introducing "guaranteed" commitments + TotalGuaranteedBefore: 0, + TotalGuaranteedAfter: 0, + Commitments: []liquid.Commitment{ + { + UUID: c.UUID, + OldStatus: oldStatus, + NewStatus: Some(liquid.CommitmentStatusConfirmed), + Amount: c.Amount, + ExpiresAt: c.ExpiresAt, + }, + }, + }, + }, + }, + }, + } + cacs := make(map[liquid.CommitmentUUID]audit.CommitmentAttributeChangeset) + var ( + potentiallyTransferredCommitmentIdxs []int + lastConsumedAmount uint64 + leftoverCommitment db.ProjectCommitment + overallTransferredAmount uint64 + ) + for idx, tc := range t.transferableCommitments { + // First, we check whether we have already transferred the full amount. if overallTransferredAmount == c.Amount { break } - // commitments cannot be consumed within the same project, mostly to avoid - // easily exploitable loopholes in the commitment confirmation process + // Commitments cannot be consumed within the same project, mostly to avoid + // easily exploitable loopholes in the commitment confirmation process. if tc.ProjectID == c.ProjectID { continue } @@ -147,67 +216,47 @@ func (t *TransferableCommitmentCache) CheckAndConsume(c db.ProjectCommitment, cu continue } // do not consume a commitment that has already been fully consumed - // NOTE: this branch will not be taken for partially consumed commitments, because `transferableCommitments` - // contains the newly spawned leftover commitment instead + // NOTE: this branch will not be taken for partially consumed commitments, because transferableCommitments + // contains the newly spawned leftover commitment instead. if _, exists := t.transferredCommitmentIDs[tc.ProjectID][tc.ID]; exists { continue } - // all checks passed, so this project gets at least one transfer - if _, exists := t.transferredCommitmentIDs[tc.ProjectID]; !exists { - t.transferredCommitmentIDs[tc.ProjectID] = make(map[db.ProjectCommitmentID]commitmentTransferLeftover) - } - // prepare audit event data - project := t.affectedProjectsByID[tc.ProjectID] - domain := t.affectedDomainsByID[project.DomainID] - auditResource := liquid.ResourceCommitmentChangeset{ - TotalConfirmedBefore: currentTotalConfirmed, - TotalConfirmedAfter: currentTotalConfirmed, // will be adjusted below based on how much is consumed - // TODO: change when introducing "guaranteed" commitments - TotalGuaranteedBefore: 0, - TotalGuaranteedAfter: 0, - Commitments: []liquid.Commitment{ - { - UUID: tc.UUID, - OldStatus: Some(tc.Status), - NewStatus: Some(liquid.CommitmentStatusSuperseded), - Amount: tc.Amount, - ConfirmBy: tc.ConfirmBy, - ExpiresAt: tc.ExpiresAt, + // commitment is considered for transfer - add it to the list + potentiallyTransferredCommitmentIdxs = append(potentiallyTransferredCommitmentIdxs, idx) + + // prep CCR structures if empty + tcProject := t.affectedProjectsByID[tc.ProjectID] + if _, exists := ccr.ByProject[tcProject.UUID]; !exists { + tcDomain := t.affectedDomainsByID[tcProject.DomainID] + ccr.ByProject[tcProject.UUID] = liquid.ProjectCommitmentChangeset{ + ProjectMetadata: LiquidProjectMetadataFromDBProject(tcProject, tcDomain), + ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ + t.loc.ResourceName: { + TotalConfirmedBefore: t.stats.ProjectStats[tc.ProjectID].Committed, + TotalConfirmedAfter: t.stats.ProjectStats[tc.ProjectID].Committed, // will be adjusted below based on how much is consumed + // TODO: change when introducing "guaranteed" commitments + TotalGuaranteedBefore: 0, + TotalGuaranteedAfter: 0, + }, }, - }, - } - t.cacs[tc.UUID] = audit.CommitmentAttributeChangeset{ - OldTransferStatus: Some(tc.TransferStatus), - NewTransferStatus: Some(limesresources.CommitmentTransferStatusNone), + } } + rcc := ccr.ByProject[tcProject.UUID].ByResource[t.loc.ResourceName] - // at least a part of this commitment will be consumed, so we will supersede it in any case + // modify CCR/ CAC structures amountToConsume := c.Amount - overallTransferredAmount if tc.Amount > amountToConsume { - // the leftover amount to be transferred is not enough to consume the whole commitment - // we will place a new commitment for the leftover amount - overallTransferredAmount += amountToConsume - leftoverCommitment, err := BuildSplitCommitment(tc, tc.Amount-amountToConsume, t.now, t.generateProjectCommitmentUUID) + // The leftover amount to be transferred is not enough to consume the whole commitment. + // We have to create a leftover for the transferable commitment. + leftoverCommitment, err = BuildSplitCommitment(*tc, tc.Amount-amountToConsume, t.now, t.generateProjectCommitmentUUID) if err != nil { - return err + return result, err } leftoverCommitment.TransferStatus = limesresources.CommitmentTransferStatusPublic leftoverCommitment.TransferToken = Some(t.generateTransferToken()) leftoverCommitment.TransferStartedAt = tc.TransferStartedAt - err = t.dbi.Insert(&leftoverCommitment) - if err != nil { - return err - } - - t.transferableCommitments[idx] = leftoverCommitment - t.transferableCommitmentsByID[leftoverCommitment.ID] = &leftoverCommitment - t.transferredCommitmentIDs[tc.ProjectID][tc.ID] = commitmentTransferLeftover{ - Amount: tc.Amount - amountToConsume, - ID: leftoverCommitment.ID, - } - - auditResource.Commitments = append(auditResource.Commitments, liquid.Commitment{ + rcc.Commitments = append(rcc.Commitments, liquid.Commitment{ UUID: leftoverCommitment.UUID, OldStatus: None[liquid.CommitmentStatus](), NewStatus: Some(leftoverCommitment.Status), @@ -215,31 +264,76 @@ func (t *TransferableCommitmentCache) CheckAndConsume(c db.ProjectCommitment, cu ConfirmBy: leftoverCommitment.ConfirmBy, ExpiresAt: leftoverCommitment.ExpiresAt, }) - if tc.Status == liquid.CommitmentStatusConfirmed { - auditResource.TotalConfirmedAfter -= amountToConsume - } + + lastConsumedAmount = amountToConsume } else { // the transferable commitment is fully consumed - overallTransferredAmount += tc.Amount - t.transferredCommitmentIDs[tc.ProjectID][tc.ID] = commitmentTransferLeftover{} + lastConsumedAmount = tc.Amount + } + overallTransferredAmount += lastConsumedAmount - if tc.Status == liquid.CommitmentStatusConfirmed { - auditResource.TotalConfirmedAfter -= tc.Amount - } + rcc.Commitments = append(rcc.Commitments, liquid.Commitment{ + UUID: tc.UUID, + OldStatus: Some(tc.Status), + NewStatus: None[liquid.CommitmentStatus](), + Amount: tc.Amount, + ConfirmBy: tc.ConfirmBy, + ExpiresAt: tc.ExpiresAt, + }) + if tc.Status == liquid.CommitmentStatusConfirmed { + rcc.TotalConfirmedAfter -= lastConsumedAmount } + ccr.ByProject[tcProject.UUID].ByResource[t.loc.ResourceName] = rcc + cacs[tc.UUID] = audit.CommitmentAttributeChangeset{ + OldTransferStatus: tc.TransferStatus, + NewTransferStatus: limesresources.CommitmentTransferStatusNone, + } + } - // retain ccr for audit event - t.ccrs[tc.UUID] = liquid.CommitmentChangeRequest{ - AZ: t.loc.AvailabilityZone, - InfoVersion: t.serviceInfo.Version, - ByProject: map[liquid.ProjectUUID]liquid.ProjectCommitmentChangeset{ - project.UUID: { - ProjectMetadata: Some(core.KeystoneProjectFromDB(project, core.KeystoneDomain{UUID: domain.UUID, Name: domain.Name}).ForLiquid()), - ByResource: map[liquid.ResourceName]liquid.ResourceCommitmentChangeset{ - t.loc.ResourceName: auditResource, - }, - }, - }, + // check that the ccr is accepted + logg.Debug("checking CanConfirmWithTransfers in %s: commitmentID = %d, projectID = %d, overall amount = %d, missing amount = %d", + t.loc.ShortScopeString(), c.ID, c.ProjectID, c.Amount, c.Amount-overallTransferredAmount) + result, err = t.delegateChangeCommitmentsWithShortcut(ctx, ccr) + if err != nil { + return result, err + } + if result.RejectionReason != "" || dryRun { + return result, nil + } + + // adjust stats locally + t.updateStats(ccr) + + // add audit events + t.auditEventsByConfirmedCommitmentUUID[c.UUID] = t.assembleAuditEvents(ccr, cacs, project.UUID, auditContext, auditAction) + + // add commitment changes to database + for i, idx := range slices.Backward(potentiallyTransferredCommitmentIdxs) { + tc := t.transferableCommitments[idx] + if _, exists := t.transferredCommitmentIDs[tc.ProjectID]; !exists { + t.transferredCommitmentIDs[tc.ProjectID] = make(map[db.ProjectCommitmentID]commitmentTransferLeftover) + } + + // delete the audit event, if the commitment was confirmed previously + if _, exists := t.auditEventsByConfirmedCommitmentUUID[c.UUID]; exists { + delete(t.auditEventsByConfirmedCommitmentUUID, tc.UUID) + } + + // Insert the leftover commitment, if exists. + if i == 0 && tc.Amount > lastConsumedAmount { + err = t.dbi.Insert(&leftoverCommitment) + if err != nil { + return result, err + } + + t.transferableCommitments[idx] = &leftoverCommitment + t.transferableCommitmentsByID[leftoverCommitment.ID] = &leftoverCommitment + t.transferredCommitmentIDs[tc.ProjectID][tc.ID] = commitmentTransferLeftover{ + Amount: tc.Amount - lastConsumedAmount, + ID: leftoverCommitment.ID, + } + } else { + t.transferredCommitmentIDs[tc.ProjectID][tc.ID] = commitmentTransferLeftover{} } // supersede consumed commitment @@ -255,18 +349,18 @@ func (t *TransferableCommitmentCache) CheckAndConsume(c db.ProjectCommitment, cu } buf, err := json.Marshal(supersedeContext) if err != nil { - return err + return result, err } tc.SupersedeContextJSON = Some(json.RawMessage(buf)) - _, err = t.dbi.Update(&tc) + _, err = t.dbi.Update(tc) if err != nil { - return err + return result, err } } - return nil + return result, nil } -// ConfirmTransferableCommitmentIfExists should be used between calls to CheckAndConsume +// ConfirmTransferableCommitmentIfExists should be used between calls to CanConfirmWithTransfers // when a commitment has been confirmed outside of the cache. The function does nothing, // when the commitment with the given ID is not in the cache. func (t *TransferableCommitmentCache) ConfirmTransferableCommitmentIfExists(id db.ProjectCommitmentID, confirmedAt time.Time) { @@ -284,13 +378,23 @@ func (t *TransferableCommitmentCache) CommitmentWasTransferred(id db.ProjectComm return exists } +// getTransferredCommitmentsForProject returns all commitments that have been transferred +// for the given project via CanConfirmWithTransfers. This can be used to deduplicate +// the transferred with the confirmed commitments when processing outside the cache. func (t *TransferableCommitmentCache) getTransferredCommitmentsForProject(projectID db.ProjectID) map[db.ProjectCommitmentID]commitmentTransferLeftover { return t.transferredCommitmentIDs[projectID] } -// GenerateAuditEventsAndMails generates the audit events and mail notifications -// for all transferred commitments that were processed via CheckAndConsume. -func (t *TransferableCommitmentCache) GenerateAuditEventsAndMails(apiIdentity core.ResourceRef, auditContext audit.Context) (auditEvents []audittools.Event, err error) { +// GenerateTransferMails generates the mail notifications for all transferred commitments +// that were processed via CanConfirmWithTransfers. For that, it collates multiple consecutive partial +// transfers to only generate a mail from the initial to the latest state. +func (t *TransferableCommitmentCache) GenerateTransferMails(apiIdentity core.ResourceRef) error { + // The system can be configured to not send mails (e.g. for test systems). + tpl, tplExists := t.mailTemplate.Unpack() + if !tplExists { + return nil + } + // first, we deduplicate the transfers per project by linking the last leftover to the first transfer commitment for _, projectID := range slices.Sorted(maps.Keys(t.transferredCommitmentIDs)) { transfers := t.transferredCommitmentIDs[projectID] @@ -311,32 +415,27 @@ func (t *TransferableCommitmentCache) GenerateAuditEventsAndMails(apiIdentity co } } - // gather the audit events and mail notifications for this project - var ( - n core.CommitmentGroupNotification - domainUUID string - projectUUID liquid.ProjectUUID - ) - err = t.dbi.QueryRow("SELECT d.uuid, d.name, p.uuid, p.name FROM domains d JOIN projects p ON d.id = p.domain_id where p.id = $1", projectID).Scan(&domainUUID, &n.DomainName, &projectUUID, &n.ProjectName) - if err != nil { - return auditEvents, err - } - - commitmentsByID, err := db.BuildIndexOfDBResult(t.dbi, func(c db.ProjectCommitment) db.ProjectCommitmentID { return c.ID }, `SELECT * FROM project_commitments WHERE id = ANY($1)`, pq.Array(slices.Collect(maps.Keys(notifiableTransfers)))) - if err != nil { - return auditEvents, err + // gather mail notifications for this project + affectedProject := t.affectedProjectsByID[projectID] + affectedDomain := t.affectedDomainsByID[affectedProject.DomainID] + n := core.CommitmentGroupNotification{ + DomainName: affectedDomain.Name, + ProjectName: affectedProject.Name, + Commitments: make([]core.CommitmentNotification, 0, len(notifiableTransfers)), } - n.Commitments = make([]core.CommitmentNotification, 0, len(notifiableTransfers)) for _, cID := range slices.Sorted(maps.Keys(notifiableTransfers)) { leftover := notifiableTransfers[cID] - c, exists := commitmentsByID[cID] + c, exists := t.transferableCommitmentsByID[cID] + // defense in depth: this should never happen if !exists { - return auditEvents, fmt.Errorf("tried to generate mail notification for non-existent commitment ID %d", cID) + return fmt.Errorf("tried to generate mail notification for non-existent commitment ID %d", cID) } - confirmedAt := c.ConfirmedAt.UnwrapOr(time.Unix(0, 0)) // the UnwrapOr() is defense in depth, it should never be relevant because we only notify for confirmed commitments here + + // also defense in depth, as all transferred commitments are superseded in CanConfirmWithTransfers + confirmedAt := c.SupersededAt.UnwrapOr(time.Unix(0, 0)) n.Commitments = append(n.Commitments, core.CommitmentNotification{ - Commitment: c, + Commitment: *c, DateString: confirmedAt.Format(time.DateOnly), Resource: core.AZResourceLocation{ ServiceType: db.ServiceType(apiIdentity.ServiceType), @@ -345,30 +444,107 @@ func (t *TransferableCommitmentCache) GenerateAuditEventsAndMails(apiIdentity co }, LeftoverAmount: leftover.Amount, }) - - // push one transfer audit event per commitment, because they belong to separate CCRs - auditEvents = append(auditEvents, audit.CommitmentEventTarget{ - CommitmentChangeRequest: t.ccrs[c.UUID], - CommitmentAttributeChangeset: map[liquid.CommitmentUUID]audit.CommitmentAttributeChangeset{c.UUID: t.cacs[c.UUID]}, - }.ReplicateForAllProjects(audittools.Event{ - Time: t.now, - Request: auditContext.Request, - User: auditContext.UserIdentity, - ReasonCode: http.StatusOK, - Action: ConsumeAction, - })...) } - if tpl, exists := t.mailTemplate.Unpack(); len(n.Commitments) != 0 && exists { - // push mail notifications + if len(n.Commitments) != 0 { + // push mail notifications to database mail, err := tpl.Render(n, projectID, t.now) if err != nil { - return auditEvents, err + return err } err = t.dbi.Insert(&mail) if err != nil { - return auditEvents, err + return err + } + } + } + return nil +} + +// RetrieveAuditEvents returns all audit events. It should be called after all calls to CanConfirmWithTransfers +// as they get deduplicated in the process. +func (t *TransferableCommitmentCache) RetrieveAuditEvents() []audittools.Event { + var events []audittools.Event + for _, evs := range t.auditEventsByConfirmedCommitmentUUID { + events = append(events, evs...) + } + return events +} + +/////////////////////////////////////////////////////////////////////////// +// internal functions: + +// assembleAuditEvents constructs the audit events for all affected projects +// with the commitments that were processed via CanConfirmWithTransfers. +func (t *TransferableCommitmentCache) assembleAuditEvents(ccr liquid.CommitmentChangeRequest, cacs map[liquid.CommitmentUUID]audit.CommitmentAttributeChangeset, consumingProjectUUID liquid.ProjectUUID, auditContext audit.Context, auditAction cadf.Action) []audittools.Event { + return audit.CommitmentEventTarget{ + CommitmentChangeRequest: ccr, + CommitmentAttributeChangesets: cacs, + }.ReplicateForAllProjects(audittools.Event{ + Time: t.now, + Request: auditContext.Request, + User: auditContext.UserIdentity, + ReasonCode: http.StatusOK, + Action: ConsumeAction, + }, Some(auditAction), Some(consumingProjectUUID)) +} + +// delegateChangeCommitmentsWithShortcut calls DelegateChangeCommitments unless we know +// that the resource does not manage commitments, in which case we can shortcut the call +// by checking the capacity locally. This way, only the local stats get used in the process. +func (t *TransferableCommitmentCache) delegateChangeCommitmentsWithShortcut(ctx context.Context, ccr liquid.CommitmentChangeRequest) (result liquid.CommitmentChangeResponse, err error) { + // optimization: we check locally, when we know that the resource does not manage commitments + // this avoids having to re-load the stats later in the callchain. + switch { + case !ccr.RequiresConfirmation(): + result = liquid.CommitmentChangeResponse{} + case !t.liquidHandlesCommitments: + behavior := t.cluster.CommitmentBehaviorForResource(t.loc.ServiceType, t.loc.ResourceName) + additions := make(map[db.ProjectID]uint64) + subtractions := make(map[db.ProjectID]uint64) + for projectUUID, pcc := range ccr.ByProject { + rcc := pcc.ByResource[t.loc.ResourceName] + affectedProject := t.affectedProjectsByUUID[projectUUID] + if rcc.TotalConfirmedAfter > rcc.TotalConfirmedBefore { + additions[affectedProject.ID] = rcc.TotalConfirmedAfter - rcc.TotalConfirmedBefore + } + if rcc.TotalConfirmedBefore > rcc.TotalConfirmedAfter { + subtractions[affectedProject.ID] = rcc.TotalConfirmedBefore - rcc.TotalConfirmedAfter + } + } + accepted := t.stats.CanAcceptCommitmentChanges(additions, subtractions, behavior) + if !accepted { + result = liquid.CommitmentChangeResponse{ + RejectionReason: "not enough capacity!", + RetryAt: None[time.Time](), + } + } + default: + commitmentChangeResponse, err := DelegateChangeCommitments(ctx, t.cluster, ccr, t.loc, t.serviceInfo, t.dbi) + if err != nil { + return result, err + } + result = commitmentChangeResponse + } + if result.RejectionReason != "" { + logg.Info("commitment not accepted for %s: %s", t.loc.ShortScopeString(), result.RejectionReason) + } + return result, nil +} + +// updateStats modifies the local stats according to the given CCR. +func (t *TransferableCommitmentCache) updateStats(ccr liquid.CommitmentChangeRequest) { + for projectUUID, pcc := range ccr.ByProject { + rcc := pcc.ByResource[t.loc.ResourceName] + affectedProject := t.affectedProjectsByUUID[projectUUID] + projectStats := t.stats.ProjectStats[affectedProject.ID] + if rcc.TotalConfirmedAfter != rcc.TotalConfirmedBefore { + newProjectStats := projectAZAllocationStats{ + Committed: rcc.TotalConfirmedAfter, + Usage: projectStats.Usage, + MinHistoricalUsage: 0, + MaxHistoricalUsage: 0, } + t.stats.ProjectStats[affectedProject.ID] = newProjectStats } } - return auditEvents, nil }