add subresource translation layer for v1 API backwards compatibility (#…

…550)

We all forgot the note that I left about this in the previous PR. :)

internal/api/fixtures/start-data-small.sql

@@ -0,0 +1,35 @@
+-- This start-data contains exactly one project and one service, with all capacity/quota/usage values at 0.
+-- It can be used as a base to set up isolated tests for individual reporting features.
+
+CREATE OR REPLACE FUNCTION unix(i integer) RETURNS timestamp AS $$ SELECT TO_TIMESTAMP(i) AT TIME ZONE 'Etc/UTC' $$ LANGUAGE SQL;
+
+INSERT INTO cluster_capacitors (capacitor_id, scraped_at, next_scrape_at) VALUES ('first', UNIX(1000), UNIX(2000));
+
+INSERT INTO cluster_services (id, type) VALUES (1, 'first');
+
+INSERT INTO cluster_resources (id, service_id, name, capacitor_id) VALUES (1, 1, 'things', 'first');
+INSERT INTO cluster_resources (id, service_id, name, capacitor_id) VALUES (2, 1, 'capacity', 'first');
+
+-- "capacity" is modeled as AZ-aware, "things" is not
+INSERT INTO cluster_az_resources (id, resource_id, az, raw_capacity, usage, subcapacities) VALUES (1, 1, 'any', 0, 0, '');
+INSERT INTO cluster_az_resources (id, resource_id, az, raw_capacity, usage, subcapacities) VALUES (2, 2, 'az-one', 0, 0, '');
+INSERT INTO cluster_az_resources (id, resource_id, az, raw_capacity, usage, subcapacities) VALUES (3, 2, 'az-two', 0, 0, '');
+
+INSERT INTO domains (id, name, uuid) VALUES (1, 'domainone', 'uuid-for-domainone');
+
+INSERT INTO projects (id, domain_id, name, uuid, parent_uuid) VALUES (1, 1, 'projectone', 'uuid-for-projectone', 'uuid-for-domainone');
+
+INSERT INTO project_services (id, project_id, type, scraped_at, rates_scraped_at, checked_at, rates_checked_at) VALUES (1, 1, 'first', UNIX(11), UNIX(12), UNIX(11), UNIX(12));
+
+INSERT INTO project_resources (id, service_id, name, quota, backend_quota) VALUES (1, 1, 'things', 0, 0);
+INSERT INTO project_resources (id, service_id, name, quota, backend_quota) VALUES (2, 1, 'capacity', 0, 0);
+INSERT INTO project_resources (id, service_id, name, quota, backend_quota) VALUES (3, 1, 'capacity_portion', NULL, NULL);
+
+-- "capacity" and "capacity_portion" are modeled as AZ-aware, "things" is not
+INSERT INTO project_az_resources (id, resource_id, az, quota, usage, physical_usage, subresources) VALUES (1, 1, 'any', 0, 0, NULL, '');
+INSERT INTO project_az_resources (id, resource_id, az, quota, usage, physical_usage, subresources) VALUES (2, 2, 'any', 0, 0, NULL, '');
+INSERT INTO project_az_resources (id, resource_id, az, quota, usage, physical_usage, subresources) VALUES (3, 2, 'az-one', 0, 0, NULL, '');
+INSERT INTO project_az_resources (id, resource_id, az, quota, usage, physical_usage, subresources) VALUES (4, 2, 'az-two', 0, 0, NULL, '');
+INSERT INTO project_az_resources (id, resource_id, az, quota, usage, physical_usage, subresources) VALUES (5, 3, 'any', NULL, 0, NULL, '');
+INSERT INTO project_az_resources (id, resource_id, az, quota, usage, physical_usage, subresources) VALUES (6, 3, 'az-one', NULL, 0, NULL, '');
+INSERT INTO project_az_resources (id, resource_id, az, quota, usage, physical_usage, subresources) VALUES (7, 3, 'az-two', NULL, 0, NULL, '');

internal/api/translation_test.go

@@ -0,0 +1,256 @@
+/*******************************************************************************
+*
+* Copyright 2024 SAP SE
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You should have received a copy of the License along with this
+* program. If not, you may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+*******************************************************************************/
+
+package api
+
+import (
+ "encoding/json"
+ "net/http"
+ "testing"
+
+ "github.com/sapcc/go-bits/assert"
+ "github.com/sapcc/go-bits/must"
+
+ "github.com/sapcc/limes/internal/core"
+ "github.com/sapcc/limes/internal/test"
+)
+
+const (
+ testSmallConfigYAML = `
+ availability_zones: [ az-one, az-two ]
+ discovery:
+ method: --test-static
+ services:
+ - service_type: first
+ type: --test-generic
+ `
+)
+
+func TestTranslateManilaSubcapacities(t *testing.T) {
+ s := test.NewSetup(t,
+ test.WithDBFixtureFile("fixtures/start-data-small.sql"),
+ test.WithConfig(testSmallConfigYAML),
+ test.WithAPIHandler(NewV1API),
+ )
+ s.Cluster.Config.ResourceBehaviors = []core.ResourceBehavior{{
+ FullResourceNameRx: "first/capacity",
+ TranslationRuleInV1API: must.Return(core.NewTranslationRule("manila")),
+ }}
+
+ // this is what liquid-manila (or liquid-cinder) writes into the DB
+ newFormatSubcapacities := []assert.JSONObject{
+ {
+ "name": "pool1",
+ "capacity": 520,
+ "usage": 520,
+ "attributes": assert.JSONObject{
+ "exclusion_reason": "hardware_state = in_decom",
+ "real_capacity": 15360,
+ },
+ },
+ {
+ "name": "pool2",
+ "capacity": 15360,
+ "usage": 10,
+ "attributes": assert.JSONObject{},
+ },
+ }
+ _, err := s.DB.Exec(`UPDATE cluster_az_resources SET subcapacities = $1 WHERE id = 2`,
+ string(must.Return(json.Marshal(newFormatSubcapacities))),
+ )
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+
+ // this is what we expect to be reported on the API
+ oldFormatSubcapacities := []assert.JSONObject{
+ {
+ "pool_name": "pool1",
+ "az": "az-one",
+ "capacity_gib": 15360,
+ "usage_gib": 520,
+ "exclusion_reason": "hardware_state = in_decom",
+ },
+ {
+ "pool_name": "pool2",
+ "az": "az-one",
+ "capacity_gib": 15360,
+ "usage_gib": 10,
+ "exclusion_reason": "",
+ },
+ }
+
+ assert.HTTPRequest{
+ Method: "GET",
+ Path: "/v1/clusters/current?resource=capacity&detail",
+ Header: map[string]string{"X-Limes-V2-API-Preview": "per-az"},
+ ExpectStatus: http.StatusOK,
+ ExpectBody: assert.JSONObject{
+ "cluster": assert.JSONObject{
+ "id": "current",
+ "min_scraped_at": 1000,
+ "max_scraped_at": 1000,
+ "services": []assert.JSONObject{{
+ "type": "first",
+ "area": "first",
+ "min_scraped_at": 11,
+ "max_scraped_at": 11,
+ "resources": []assert.JSONObject{{
+ "name": "capacity",
+ "unit": "B",
+ "capacity": 0,
+ "domains_quota": 0,
+ "usage": 0,
+ "per_availability_zone": []assert.JSONObject{
+ {"capacity": 0, "name": "az-one"},
+ {"capacity": 0, "name": "az-two"},
+ },
+ "per_az": assert.JSONObject{
+ "az-one": assert.JSONObject{"capacity": 0, "usage": 0, "subcapacities": oldFormatSubcapacities},
+ "az-two": assert.JSONObject{"capacity": 0, "usage": 0},
+ },
+ "quota_distribution_model": "autogrow",
+ "subcapacities": oldFormatSubcapacities,
+ }},
+ }},
+ },
+ },
+ }.Check(t, s.Handler)
+}
+
+func TestTranslateCinderVolumeSubresources(t *testing.T) {
+ subresourcesInLiquidFormat := []assert.JSONObject{
+ {
+ "id": "6dfbbce3-078d-4c64-8f88-8145b8d44183",
+ "name": "volume1",
+ "attributes": assert.JSONObject{
+ "size_gib": 21,
+ "status": "error",
+ },
+ },
+ {
+ "id": "a33bae62-e14e-47a6-a019-5faf89c20dc7",
+ "name": "volume2",
+ "attributes": assert.JSONObject{
+ "size_gib": 1,
+ "status": "available",
+ },
+ },
+ }
+
+ subresourcesInLegacyFormat := []assert.JSONObject{
+ {
+ "id": "6dfbbce3-078d-4c64-8f88-8145b8d44183",
+ "name": "volume1",
+ "status": "error",
+ "size": assert.JSONObject{"value": 21, "unit": "GiB"},
+ "availability_zone": "az-one",
+ },
+ {
+ "id": "a33bae62-e14e-47a6-a019-5faf89c20dc7",
+ "name": "volume2",
+ "status": "available",
+ "size": assert.JSONObject{"value": 1, "unit": "GiB"},
+ "availability_zone": "az-one",
+ },
+ }
+
+ testSubresourceTranslation(t, "cinder-volumes", subresourcesInLiquidFormat, subresourcesInLegacyFormat)
+}
+
+func TestTranslateCinderSnapshotSubresources(t *testing.T) {
+ subresourcesInLiquidFormat := []assert.JSONObject{
+ {
+ "id": "260da0ee-4816-48af-8784-1717cb76c0cd",
+ "name": "snapshot1-of-volume2",
+ "attributes": assert.JSONObject{
+ "size_gib": 1,
+ "status": "available",
+ "volume_id": "a33bae62-e14e-47a6-a019-5faf89c20dc7",
+ },
+ },
+ }
+
+ subresourcesInLegacyFormat := []assert.JSONObject{
+ {
+ "id": "260da0ee-4816-48af-8784-1717cb76c0cd",
+ "name": "snapshot1-of-volume2",
+ "status": "available",
+ "size": assert.JSONObject{"value": 1, "unit": "GiB"},
+ "volume_id": "a33bae62-e14e-47a6-a019-5faf89c20dc7",
+ },
+ }
+
+ testSubresourceTranslation(t, "cinder-snapshots", subresourcesInLiquidFormat, subresourcesInLegacyFormat)
+}
+
+func testSubresourceTranslation(t *testing.T, ruleID string, subresourcesInLiquidFormat, subresourcesInLegacyFormat []assert.JSONObject) {
+ s := test.NewSetup(t,
+ test.WithDBFixtureFile("fixtures/start-data-small.sql"),
+ test.WithConfig(testSmallConfigYAML),
+ test.WithAPIHandler(NewV1API),
+ )
+ s.Cluster.Config.ResourceBehaviors = []core.ResourceBehavior{{
+ FullResourceNameRx: "first/capacity",
+ TranslationRuleInV1API: must.Return(core.NewTranslationRule(ruleID)),
+ }}
+
+ _, err := s.DB.Exec(`UPDATE project_az_resources SET subresources = $1 WHERE id = 3`,
+ string(must.Return(json.Marshal(subresourcesInLiquidFormat))),
+ )
+ if err != nil {
+ t.Fatal(err.Error())
+ }
+
+ assert.HTTPRequest{
+ Method: "GET",
+ Path: "/v1/domains/uuid-for-domainone/projects/uuid-for-projectone?resource=capacity&detail",
+ Header: map[string]string{"X-Limes-V2-API-Preview": "per-az"},
+ ExpectStatus: http.StatusOK,
+ ExpectBody: assert.JSONObject{
+ "project": assert.JSONObject{
+ "id": "uuid-for-projectone",
+ "name": "projectone",
+ "parent_id": "uuid-for-domainone",
+ "services": []assert.JSONObject{
+ {
+ "type": "first",
+ "area": "first",
+ "scraped_at": 11,
+ "resources": []assert.JSONObject{
+ {
+ "name": "capacity",
+ "unit": "B",
+ "quota": 0,
+ "usable_quota": 0,
+ "usage": 0,
+ "per_az": assert.JSONObject{
+ "az-one": assert.JSONObject{"quota": 0, "usage": 0, "subresources": subresourcesInLegacyFormat},
+ "az-two": assert.JSONObject{"quota": 0, "usage": 0},
+ },
+ "quota_distribution_model": "autogrow",
+ "subresources": subresourcesInLegacyFormat,
+ },
+ },
+ },
+ },
+ },
+ },
+ }.Check(t, s.Handler)
+}

internal/core/resource_behavior.go

@@ -42,6 +42,7 @@ type ResourceBehavior struct {
  CommitmentUntilPercent *float64 `yaml:"commitment_until_percent"`
  CommitmentConversion CommitmentConversion `yaml:"commitment_conversion"`
  IdentityInV1API ResourceRef `yaml:"identity_in_v1_api"`
+ TranslationRuleInV1API TranslationRule `yaml:"translation_rule_in_v1_api"`
  Category string `yaml:"category"`
 }
 
@@ -115,6 +116,9 @@ func (b *ResourceBehavior) Merge(other ResourceBehavior) {
  if other.IdentityInV1API != (ResourceRef{}) {
  b.IdentityInV1API = other.IdentityInV1API
  }
+ if !other.TranslationRuleInV1API.IsEmpty() {
+ b.TranslationRuleInV1API = other.TranslationRuleInV1API
+ }
  if other.Category != "" {
  b.Category = other.Category
  }