don't cache the export index (#494)

For those blob storage implementations that support it, pass
in an optional bool for whether a created object should be
cacheable, and have the worker set that to false for index.txt.

internal/export/worker.go

@@ -205,7 +205,7 @@ func (s *Server) createFile(ctx context.Context, cfi createFileInfo) (string, er
  logger.Infof("Created file %v, signed with %v keys", objectName, len(signers))
  ctx, cancel := context.WithTimeout(ctx, blobOperationTimeout)
  defer cancel()
- if err := s.env.Blobstore().CreateObject(ctx, cfi.exportBatch.BucketName, objectName, data); err != nil {
+ if err := s.env.Blobstore().CreateObject(ctx, cfi.exportBatch.BucketName, objectName, data, false); err != nil {
  return "", fmt.Errorf("creating file %s in bucket %s: %w", objectName, cfi.exportBatch.BucketName, err)
  }
  return objectName, nil
@@ -276,7 +276,7 @@ func (s *Server) createIndex(ctx context.Context, eb *model.ExportBatch, newObje
  indexObjectName := exportIndexFilename(eb)
  ctx, cancel := context.WithTimeout(ctx, blobOperationTimeout)
  defer cancel()
- if err := s.env.Blobstore().CreateObject(ctx, eb.BucketName, indexObjectName, data); err != nil {
+ if err := s.env.Blobstore().CreateObject(ctx, eb.BucketName, indexObjectName, data, false); err != nil {
  return "", 0, fmt.Errorf("creating file %s in bucket %s: %w", indexObjectName, eb.BucketName, err)
  }
  return indexObjectName, len(objects), nil

internal/storage/filesystem_storage.go

@@ -38,7 +38,7 @@ func NewFilesystemStorage(ctx context.Context) (Blobstore, error) {
 
 // CreateObject creates a new object on the filesystem or overwrites an existing
 // one.
-func (s *FilesystemStorage) CreateObject(ctx context.Context, folder, filename string, contents []byte) error {
+func (s *FilesystemStorage) CreateObject(ctx context.Context, folder, filename string, contents []byte, cacheable bool) error {
  pth := filepath.Join(folder, filename)
  if err := ioutil.WriteFile(pth, contents, 0644); err != nil {
  return fmt.Errorf("failed to create object: %w", err)

internal/storage/filesystem_storage_test.go

@@ -63,7 +63,7 @@ func TestFilesystemStorage_CreateObject(t *testing.T) {
  t.Fatal(err)
  }
 
- err = storage.CreateObject(ctx, tc.folder, tc.filepath, tc.contents)
+ err = storage.CreateObject(ctx, tc.folder, tc.filepath, tc.contents, false)
  if (err != nil) != tc.err {
  t.Fatal(err)
  }

internal/storage/google_cloud_storage.go

@@ -43,8 +43,13 @@ func NewGoogleCloudStorage(ctx context.Context) (Blobstore, error) {
 }
 
 // CreateObject creates a new cloud storage object or overwrites an existing one.
-func (gcs *GoogleCloudStorage) CreateObject(ctx context.Context, bucket, objectName string, contents []byte) error {
+func (gcs *GoogleCloudStorage) CreateObject(ctx context.Context, bucket, objectName string, contents []byte, cacheable bool) error {
  wc := gcs.client.Bucket(bucket).Object(objectName).NewWriter(ctx)
+ if !cacheable {
+ wc.Metadata = map[string]string{
+ "Cache-Control": "no-cache, max-age=0",
+ }
+ }
  if _, err := wc.Write(contents); err != nil {
  return fmt.Errorf("storage.Writer.Write: %w", err)
  }

internal/storage/noop_storage.go

@@ -26,7 +26,7 @@ func NewNoopBlobstore(ctx context.Context) (Blobstore, error) {
  return &NoopBlobstore{}, nil
 }
 
-func (s *NoopBlobstore) CreateObject(ctx context.Context, folder, filename string, contents []byte) error {
+func (s *NoopBlobstore) CreateObject(ctx context.Context, folder, filename string, contents []byte, cacheable bool) error {
  return nil
 }
 

internal/storage/storage.go

@@ -37,7 +37,7 @@ type Config struct {
 // Blobstore defines the minimum interface for a blob storage system.
 type Blobstore interface {
  // CreateObject creates or overwrites an object in the storage system.
- CreateObject(ctx context.Context, bucket, objectName string, contents []byte) error
+ CreateObject(ctx context.Context, bucket, objectName string, contents []byte, cacheable bool) error
 
  // DeleteObject deltes an object or does nothing if the object doesn't exist.
  DeleteObject(ctx context.Context, bucket, objectName string) error