feat: Saving Graphql mutation object to Persistent Volume (#3460)

* Initial version of graphql middleware

* Saving file to Persistent Volume

* Minor fixes

* change the way of saving Gardener Shoot to PV

* delete unnecessary files

* fix naming

* Added storing graphQL query

* Compilation error

* Compilation error

* Compilation error

* Minor refactor

* Unit test fix

* Functions for dumping spec and graphql moved to testkit package

* Fixes in GraphQL dump

* Fixes in GraphQL dump

* Added logging

* Added logs for debugging

* Fixed issue with dumping GraphQL

* Fixed issue with dumping GraphQL

* Introduced TestDataWriter

* Compilation fix

* Minor refactoring

* Names of the dumped files aligned with KIM

---------

Co-authored-by: Arkadiusz Galwas <[email protected]>

components/provisioner/cmd/main.go

@@ -3,6 +3,7 @@ package main
 import (
  "context"
  "fmt"
+ "github.com/kyma-project/control-plane/components/provisioner/internal/util/testkit"
  "net/http"
  "sync"
  "time"
@@ -157,14 +158,16 @@ func main() {
 
  k8sClientProvider := k8s.NewK8sClientProvider()
 
+ testDataWriter := testkit.NewTestDataWriter(gardenerNamespace, "/testdata/provisioner", cfg.Gardener.EnableDumpShootSpec)
+
  adminKubeconfigRequest := gardenerClient.SubResource("adminkubeconfig")
  kubeconfigProvider := gardener.NewKubeconfigProvider(shootClient, adminKubeconfigRequest, secretsInterface)
 
  provisioningQueue := queue.CreateProvisioningQueue(cfg.ProvisioningTimeout, dbsFactory, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
  shootUpgradeQueue := queue.CreateShootUpgradeQueue(cfg.ProvisioningTimeout, dbsFactory, shootClient, cfg.OperatorRoleBinding, k8sClientProvider, kubeconfigProvider)
  deprovisioningQueue := queue.CreateDeprovisioningQueue(cfg.DeprovisioningTimeout, dbsFactory, shootClient)
 
- provisioner := gardener.NewProvisioner(gardenerNamespace, shootClient, dbsFactory, cfg.Gardener.AuditLogsPolicyConfigMap, cfg.Gardener.MaintenanceWindowConfigPath, cfg.Gardener.EnableDumpShootSpec)
+ provisioner := gardener.NewProvisioner(gardenerNamespace, shootClient, dbsFactory, cfg.Gardener.AuditLogsPolicyConfigMap, cfg.Gardener.MaintenanceWindowConfigPath, testDataWriter)
  shootController, err := newShootController(gardenerNamespace, gardenerClusterConfig, dbsFactory, cfg.Gardener.AuditLogsTenantConfigPath)
  exitOnError(err, "Failed to create Shoot controller.")
  go func() {
@@ -188,7 +191,7 @@ func main() {
 
  tenantUpdater := api.NewTenantUpdater(dbsFactory.NewReadWriteSession())
  validator := api.NewValidator()
- resolver := api.NewResolver(provisioningSVC, validator, tenantUpdater)
+ resolver := api.NewResolver(provisioningSVC, validator, tenantUpdater, testDataWriter)
 
  ctx, cancel := context.WithCancel(context.Background())
  defer cancel()

components/provisioner/internal/api/resolver.go

@@ -3,7 +3,6 @@ package api
 import (
  "context"
  "fmt"
-
  "github.com/kyma-project/control-plane/components/provisioner/internal/api/middlewares"
  "github.com/pkg/errors"
 
@@ -14,31 +13,40 @@ import (
 )
 
 type Resolver struct {
- provisioning provisioning.Service
- validator Validator
- tenantUpdater TenantUpdater
+ provisioning provisioning.Service
+ validator Validator
+ tenantUpdater TenantUpdater
+ testDataWriter InputDataWriter
+}
+
+type InputDataWriter interface {
+ PersistGraphQL(mutation gqlschema.ProvisionRuntimeInput) (string, error)
+ Enabled() bool
 }
 
 func (r *Resolver) Mutation() gqlschema.MutationResolver {
  return &Resolver{
- provisioning: r.provisioning,
- validator: r.validator,
- tenantUpdater: r.tenantUpdater,
+ provisioning: r.provisioning,
+ validator: r.validator,
+ tenantUpdater: r.tenantUpdater,
+ testDataWriter: r.testDataWriter,
  }
 }
 func (r *Resolver) Query() gqlschema.QueryResolver {
  return &Resolver{
- provisioning: r.provisioning,
- validator: r.validator,
- tenantUpdater: r.tenantUpdater,
+ provisioning: r.provisioning,
+ validator: r.validator,
+ tenantUpdater: r.tenantUpdater,
+ testDataWriter: r.testDataWriter,
  }
 }
 
-func NewResolver(provisioningService provisioning.Service, validator Validator, tenantUpdater TenantUpdater) *Resolver {
+func NewResolver(provisioningService provisioning.Service, validator Validator, tenantUpdater TenantUpdater, testDataWriter InputDataWriter) *Resolver {
  return &Resolver{
- provisioning: provisioningService,
- validator: validator,
- tenantUpdater: tenantUpdater,
+ provisioning: provisioningService,
+ validator: validator,
+ tenantUpdater: tenantUpdater,
+ testDataWriter: testDataWriter,
  }
 }
 
@@ -59,6 +67,17 @@ func (r *Resolver) ProvisionRuntime(ctx context.Context, config gqlschema.Provis
 
  log.Infof("Requested provisioning of Runtime %s.", config.RuntimeInput.Name)
 
+ if r.testDataWriter.Enabled() {
+ log.Infof("Saving GraphQL query for Runtime %s", config.RuntimeInput.Name)
+ path, err := r.testDataWriter.PersistGraphQL(config)
+
+ if err == nil {
+ log.Infof("GraphQL query dumped to %s", path)
+ } else {
+ log.Errorf("Failed to dump GraphQL mutation for Runtime %s: %s", config.RuntimeInput.Name, err)
+ }
+ }
+
  operationStatus, err := r.provisioning.ProvisionRuntime(config, tenant, subAccount)
  if err != nil {
  log.Errorf("Failed to provision Runtime %s: %s", config.RuntimeInput.Name, err)

components/provisioner/internal/api/resolver_integration_with_gardener_test.go

@@ -4,6 +4,8 @@ import (
  "context"
  "encoding/json"
  "fmt"
+ "github.com/kyma-project/control-plane/components/provisioner/internal/util/testkit"
+ "os"
  "path/filepath"
  "strings"
  "testing"
@@ -180,7 +182,10 @@ func TestProvisioning_ProvisionRuntimeWithDatabase(t *testing.T) {
  //uuidGeneratorMock.On("New").Return(config.upgradeID).Once()
  //uuidGeneratorMock.On("New").Return(config.deprovisioningID).Once()
 
- provisioner := gardener.NewProvisioner(namespace, shootInterface, dbsFactory, auditLogPolicyCMName, maintenanceWindowConfigPath, false)
+ tmpDir, err := os.MkdirTemp("", "")
+ require.NoError(t, err)
+
+ provisioner := gardener.NewProvisioner(namespace, shootInterface, dbsFactory, auditLogPolicyCMName, maintenanceWindowConfigPath, testkit.NewTestDataWriter("kyma-dev", tmpDir, true))
 
  inputConverter := provisioning.NewInputConverter(uuidGeneratorMock, "Project", defaultEnableKubernetesVersionAutoUpdate, defaultEnableMachineImageVersionAutoUpdate, defaultEnableIMDSv2)
  graphQLConverter := provisioning.NewGraphQLConverter()
@@ -201,7 +206,7 @@ func TestProvisioning_ProvisionRuntimeWithDatabase(t *testing.T) {
 
  tenantUpdater := api.NewTenantUpdater(dbsFactory.NewReadWriteSession())
 
- resolver := api.NewResolver(provisioningService, validator, tenantUpdater)
+ resolver := api.NewResolver(provisioningService, validator, tenantUpdater, testkit.NewTestDataWriter("kyma-dev", tmpDir, true))
 
  fullConfig := gqlschema.ProvisionRuntimeInput{RuntimeInput: &runtimeInput, ClusterConfig: &clusterConfig}
 

components/provisioner/internal/api/resolver_test.go

@@ -2,6 +2,7 @@ package api_test
 
 import (
  "context"
+ "github.com/kyma-project/control-plane/components/provisioner/internal/util/testkit"
  "testing"
 
  "github.com/kyma-project/control-plane/components/provisioner/internal/apperrors"
@@ -61,7 +62,7 @@ func TestResolver_ProvisionRuntime(t *testing.T) {
  provisioningService := &mocks.Service{}
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
- resolver := api.NewResolver(provisioningService, validator, tenantUpdater)
+ resolver := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  tenantUpdater.On("GetTenant", ctx).Return(tenant, nil)
 
@@ -112,7 +113,7 @@ func TestResolver_ProvisionRuntime(t *testing.T) {
  provisioningService := &mocks.Service{}
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  kymaConfig := &gqlschema.KymaConfigInput{
  Version: "1.5",
@@ -136,7 +137,7 @@ func TestResolver_ProvisionRuntime(t *testing.T) {
  provisioningService := &mocks.Service{}
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  kymaConfig := &gqlschema.KymaConfigInput{
  Version: "1.5",
@@ -168,7 +169,7 @@ func TestResolver_ProvisionRuntime(t *testing.T) {
  provisioningService := &mocks.Service{}
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  kymaConfig := &gqlschema.KymaConfigInput{
  Version: "1.5",
@@ -208,7 +209,7 @@ func TestResolver_DeprovisionRuntime(t *testing.T) {
  provisioningService := &mocks.Service{}
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  expectedID := "ec781980-0533-4098-aab7-96b535569732"
 
@@ -228,7 +229,7 @@ func TestResolver_DeprovisionRuntime(t *testing.T) {
  provisioningService := &mocks.Service{}
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
  provisioningService.On("DeprovisionRuntime", runtimeID).Return("", apperrors.Internal("Deprovisioning fails because reasons"))
  tenantUpdater.On("GetAndUpdateTenant", runtimeID, ctx).Return(nil)
 
@@ -246,7 +247,7 @@ func TestResolver_DeprovisionRuntime(t *testing.T) {
  provisioningService := &mocks.Service{}
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
  expectedID := "ec781980-0533-4098-aab7-96b535569732"
 
  ctx := context.Background()
@@ -273,7 +274,7 @@ func TestResolver_RuntimeStatus(t *testing.T) {
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
 
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  operationID := "acc5040c-3bb6-47b8-8651-07f6950bd0a7"
  message := "some message"
@@ -307,7 +308,7 @@ func TestResolver_RuntimeStatus(t *testing.T) {
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
 
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  provisioningService.On("RuntimeStatus", runtimeID).Return(nil, apperrors.Internal("Runtime status fails"))
  tenantUpdater.On("GetAndUpdateTenant", runtimeID, ctx).Return(nil)
@@ -332,7 +333,7 @@ func TestResolver_RuntimeOperationStatus(t *testing.T) {
  validator := &validatorMocks.Validator{}
  tenantUpdater := &validatorMocks.TenantUpdater{}
 
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  operationID := "acc5040c-3bb6-47b8-8651-07f6950bd0a7"
  message := "some message"
@@ -363,7 +364,7 @@ func TestResolver_RuntimeOperationStatus(t *testing.T) {
  tenantUpdater := &validatorMocks.TenantUpdater{}
 
  validator.On("ValidateTenantForOperation", operationID, tenant).Return(nil)
- provisioner := api.NewResolver(provisioningService, validator, tenantUpdater)
+ provisioner := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  provisioningService.On("RuntimeOperationStatus", operationID).Return(nil, apperrors.Internal("Some error"))
  tenantUpdater.On("GetAndUpdateTenant", runtimeID, ctx).Return(nil)
@@ -401,7 +402,7 @@ func TestResolver_UpgradeShoot(t *testing.T) {
  validator.On("ValidateUpgradeShootInput", upgradeShootInput).Return(nil)
  provisioningService.On("UpgradeGardenerShoot", runtimeID, upgradeShootInput).Return(operation, nil)
 
- resolver := api.NewResolver(provisioningService, validator, tenantUpdater)
+ resolver := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  //when
  status, err := resolver.UpgradeShoot(ctx, runtimeID, upgradeShootInput)
@@ -423,7 +424,7 @@ func TestResolver_UpgradeShoot(t *testing.T) {
  validator.On("ValidateUpgradeShootInput", upgradeShootInput).Return(apperrors.BadRequest("error"))
  tenantUpdater.On("GetAndUpdateTenant", runtimeID, ctx).Return(nil)
 
- resolver := api.NewResolver(provisioningService, validator, tenantUpdater)
+ resolver := api.NewResolver(provisioningService, validator, tenantUpdater, &testkit.TestDataWriter{})
 
  //when
  _, err := resolver.UpgradeShoot(ctx, runtimeID, upgradeShootInput)

components/provisioner/internal/gardener/provisioner.go

@@ -4,13 +4,11 @@ import (
  "context"
  "encoding/json"
  "fmt"
- "io"
  "os"
  "time"
 
  v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
- gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
  "github.com/kyma-project/control-plane/components/provisioner/internal/apperrors"
  "github.com/kyma-project/control-plane/components/provisioner/internal/util"
  "github.com/mitchellh/mapstructure"
@@ -24,8 +22,6 @@ import (
  "github.com/kyma-project/control-plane/components/provisioner/internal/provisioning/persistence/dbsession"
  log "github.com/sirupsen/logrus"
  k8sErrors "k8s.io/apimachinery/pkg/api/errors"
-
- "sigs.k8s.io/yaml"
 )
 
 //go:generate mockery --name=Client
@@ -35,20 +31,25 @@ type Client interface {
  Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.Shoot, err error)
 }
 
+type OutputDataWriter interface {
+ PersistShoot(shoot *v1beta1.Shoot) (string, error)
+ Enabled() bool
+}
+
 func NewProvisioner(
  namespace string,
  shootClient Client,
  factory dbsession.Factory,
  policyConfigMapName string,
  maintenanceWindowConfigPath string,
- enableDumpShootSpec bool) *GardenerProvisioner {
+ testDataWriter OutputDataWriter) *GardenerProvisioner {
  return &GardenerProvisioner{
  namespace: namespace,
  shootClient: shootClient,
  dbSessionFactory: factory,
  policyConfigMapName: policyConfigMapName,
  maintenanceWindowConfigPath: maintenanceWindowConfigPath,
- enableDumpShootSpec: enableDumpShootSpec,
+ testDataWriter:  testDataWriter,
  }
 }
 
@@ -58,7 +59,7 @@ type GardenerProvisioner struct {
  dbSessionFactory dbsession.Factory
  policyConfigMapName string
  maintenanceWindowConfigPath string
- enableDumpShootSpec bool
+ testDataWriter  OutputDataWriter
 }
 
 func (g *GardenerProvisioner) ProvisionCluster(cluster model.Cluster, operationId string) apperrors.AppError {
@@ -98,14 +99,15 @@ func (g *GardenerProvisioner) ProvisionCluster(cluster model.Cluster, operationI
  g.applyAuditConfig(shootTemplate)
  }
 
- if g.enableDumpShootSpec {
- path := fmt.Sprintf("%s/%s-%s.yaml", "/testdata/provisioner", shootTemplate.Namespace, shootTemplate.Name)
- if err := persist(path, shootTemplate); err != nil {
+ if g.testDataWriter.Enabled() {
+ log.Infof("Saving Shoot spec for %s Runtime", cluster.ID)
+ path, err := g.testDataWriter.PersistShoot(shootTemplate)
+
+ if err == nil {
+ log.Infof("Shoot spec dumped to %s", path)
+ } else {
  log.Errorf("Error marshaling Shoot spec: %s", err.Error())
  }
- log.Infof("Shoot spec dumped to %s", path)
- } else {
- log.Infof("Shoot Spec Dump feature is disabled")
  }
 
  _, k8serr := g.shootClient.Create(context.Background(), shootTemplate, v1.CreateOptions{})
@@ -117,31 +119,6 @@ func (g *GardenerProvisioner) ProvisionCluster(cluster model.Cluster, operationI
  return nil
 }
 
-var getWriter = func(filePath string) (io.Writer, error) {
- file, err := os.Create(filePath)
- if err != nil {
- return nil, fmt.Errorf("unable to create file: %w", err)
- }
- return file, nil
-}
-
-func persist(path string, s *gardener.Shoot) error {
- writer, err := getWriter(path)
- if err != nil {
- return fmt.Errorf("unable to create file: %w", err)
- }
-
- b, err := yaml.Marshal(s)
- if err != nil {
- return fmt.Errorf("unable to marshal shoot: %w", err)
- }
-
- if _, err = writer.Write(b); err != nil {
- return fmt.Errorf("unable to write to file: %w", err)
- }
- return nil
-}
-
 func (g *GardenerProvisioner) UpgradeCluster(clusterID string, upgradeConfig model.GardenerConfig) apperrors.AppError {
  err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
  shoot, err := g.shootClient.Get(context.Background(), upgradeConfig.Name, v1.GetOptions{})