Adding Publish - Export integration test.

internal/integration/en_api.go

@@ -0,0 +1,100 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package integration contains EN Server integration tests.
+package integration
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "testing"
+
+ verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1alpha1"
+)
+
+// EnServerClient provides Exposure Notifications API client to support integration testing.
+type EnServerClient struct {
+ client *http.Client
+}
+
+func (server EnServerClient) getRequest(url string) (*http.Response, error) {
+ resp, err := server.client.Get(url)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ return unwrapResponse(resp)
+}
+
+// Posts requests to the specified url.
+// This methods attempts to serialize data argument as a json.
+func (server EnServerClient) postRequest(url string, data interface{}) (*http.Response, error) {
+ jsonData, err := json.Marshal(data)
+ if err != nil {
+ return nil, fmt.Errorf("unable to marshal json payload")
+ }
+ request := bytes.NewBuffer(jsonData)
+ r, err := http.NewRequest("POST", url, request)
+ if err != nil {
+ return nil, err
+ }
+ r.Header.Set("Content-Type", "application/json")
+ resp, err := server.client.Do(r)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ return unwrapResponse(resp)
+}
+
+func (server EnServerClient) PublishKeys(t *testing.T, request verifyapi.Publish) {
+ resp, err := server.postRequest("/publish", request)
+ if err != nil {
+ t.Fatalf("request failed: %v, %v", err, resp)
+ }
+ log.Printf("response: %v", resp.Status)
+ t.Logf("Publish request is sent to %v", "/publish")
+}
+
+func (server EnServerClient) ExportBatches(t *testing.T) {
+ resp, err := server.getRequest("/export/create-batches")
+ if err != nil {
+ t.Fatalf("request failed: %v, %v", err, resp)
+ }
+ t.Logf("Create batches request is sent to %v", "/export/create-batches")
+}
+
+func (server EnServerClient) StartExportWorkers(t *testing.T) {
+ resp, err := server.getRequest("/export/do-work")
+ if err != nil {
+ t.Fatalf("request failed: %v, %v", err, resp)
+ }
+ t.Logf("Export worker request is sent to %v", "/export/do-work")
+}
+
+func unwrapResponse(resp *http.Response) (*http.Response, error) {
+ if resp.StatusCode != http.StatusOK {
+ // Return error upstream.
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to copy error body (%d): %w", resp.StatusCode, err)
+ }
+ return resp, fmt.Errorf("request failed with status: %v\n%v", resp.StatusCode, body)
+ }
+ return resp, nil
+}

internal/integration/integration_test.go

@@ -103,7 +103,7 @@ func testServer(tb testing.TB) (*serverenv.ServerEnv, *http.Client) {
  MinRecords: 1,
  PaddingRange: 1,
  MaxRecords: 10000,
- TruncateWindow: 1 * time.Second,
+ TruncateWindow: 1 * time.Millisecond,
  MinWindowAge: 1 * time.Second,
  TTL: 336 * time.Hour,
  }
@@ -130,7 +130,7 @@ func testServer(tb testing.TB) (*serverenv.ServerEnv, *http.Client) {
  MinRequestDuration: 50 * time.Millisecond,
  MaxKeysOnPublish: 15,
  MaxIntervalAge: 360 * time.Hour,
- TruncateWindow: 1 * time.Hour,
+ TruncateWindow: 1 * time.Second,
  DebugAPIResponses: true,
  DebugReleaseSameDayKeys: true,
  }

internal/integration/publish_test.go

@@ -15,35 +15,42 @@
 package integration
 
 import (
- "bytes"
  "context"
  "encoding/json"
- "net/http"
+ "strings"
  "testing"
  "time"
 
  authorizedappmodel "github.com/google/exposure-notifications-server/internal/authorizedapp/model"
+ exportapi "github.com/google/exposure-notifications-server/internal/export"
  exportdatabase "github.com/google/exposure-notifications-server/internal/export/database"
  exportmodel "github.com/google/exposure-notifications-server/internal/export/model"
+ "github.com/google/exposure-notifications-server/internal/pb/export"
  publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
  publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
+ "github.com/google/exposure-notifications-server/internal/serverenv"
  "github.com/google/exposure-notifications-server/internal/util"
  verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1alpha1"
+ "github.com/google/go-cmp/cmp"
+ "github.com/google/go-cmp/cmp/cmpopts"
+ "google.golang.org/protobuf/proto"
 )
 
 func TestPublish(t *testing.T) {
  t.Parallel()
 
  ctx := context.Background()
+
  env, client := testServer(t)
  db := env.Database()
+ enClient := &EnServerClient{client: client}
 
  // Create an authorized app.
  aa := env.AuthorizedAppProvider()
  if err := aa.Add(ctx, &authorizedappmodel.AuthorizedApp{
  AppPackageName: "com.example.app",
  AllowedRegions: map[string]struct{}{
- "US": {},
+ "TEST": {},
  },
  AllowedHealthAuthorityIDs: map[int64]struct{}{
  12345: {},
@@ -66,42 +73,29 @@ func TestPublish(t *testing.T) {
  }
 
  // Create an export config.
+ exportPeriod := 2 * time.Second
  ec := &exportmodel.ExportConfig{
  BucketName: "my-bucket",
- Period: 1 * time.Second,
- OutputRegion: "US",
+ Period: exportPeriod,
+ OutputRegion: "TEST",
  From: time.Now().Add(-2 * time.Second),
  Thru: time.Now().Add(1 * time.Hour),
- SignatureInfoIDs: []int64{si.ID},
+ SignatureInfoIDs: []int64{},
  }
  if err := exportdatabase.New(db).AddExportConfig(ctx, ec); err != nil {
  t.Fatal(err)
  }
 
- payload := &verifyapi.Publish{
+ payload := verifyapi.Publish{
  Keys: util.GenerateExposureKeys(3, -1, false),
- Regions: []string{"US"},
+ Regions: []string{"TEST"},
  AppPackageName: "com.example.app",
 
  // TODO: hook up verification
  VerificationPayload: "TODO",
  }
 
- var body bytes.Buffer
- if err := json.NewEncoder(&body).Encode(payload); err != nil {
- t.Fatal(err)
- }
-
- resp, err := client.Post("/publish", "application/json", &body)
- if err != nil {
- t.Fatal(err)
- }
- defer resp.Body.Close()
-
- // Ensure we get a successful response code.
- if got, want := resp.StatusCode, http.StatusOK; got != want {
- t.Errorf("expected %v to be %v", got, want)
- }
+ enClient.PublishKeys(t, payload)
 
  // Look up the exposures in the database.
  criteria := publishdb.IterateExposuresCriteria{
@@ -120,25 +114,104 @@ func TestPublish(t *testing.T) {
  t.Errorf("expected %v to be %v: %#v", got, want, exposures)
  }
 
- // Create an export.
- resp, err = client.Get("/export/create-batches")
+ t.Logf("Waiting %v before export batches", exportPeriod+1*time.Second)
+ time.Sleep(exportPeriod + 1*time.Second)
+ enClient.ExportBatches(t)
+
+ t.Logf("Waiting %v before starting workers", 500*time.Millisecond)
+ time.Sleep(500 * time.Millisecond)
+ enClient.StartExportWorkers(t)
+
+ keyExport := getKeysFromLatestBatch(t, "my-bucket", ctx, env)
+
+ got := keyExport
+
+ wantedKeysMap := make(map[string]*export.TemporaryExposureKey)
+ for _, key := range payload.Keys {
+ wantedKeysMap[key.Key] = &export.TemporaryExposureKey{
+ KeyData: util.DecodeKey(key.Key),
+ TransmissionRiskLevel: proto.Int32(int32(key.TransmissionRisk)),
+ RollingStartIntervalNumber: proto.Int32(key.IntervalNumber),
+ }
+ }
+
+ want := &export.TemporaryExposureKeyExport{
+ Region: proto.String("TEST"),
+ BatchNum: proto.Int32(1),
+ BatchSize: proto.Int32(1),
+ }
+
+ if *want.BatchSize != *got.BatchSize {
+ t.Errorf("Invalid BatchSize: want: %v, got: %v", *want.BatchSize, *got.BatchSize)
+ }
+
+ if *want.BatchNum != *got.BatchNum {
+ t.Errorf("Invalid BatchNum: want: %v, got: %v", *want.BatchNum, *got.BatchNum)
+ }
+
+ if *want.Region != *got.Region {
+ t.Errorf("Invalid Region: want: %v, got: %v", *want.BatchSize, *got.BatchSize)
+ }
+
+ for _, key := range got.Keys {
+ s := util.ToBase64(key.KeyData)
+ wantedKey := wantedKeysMap[s]
+ diff := cmp.Diff(wantedKey, key, cmpopts.IgnoreUnexported(export.TemporaryExposureKey{}))
+ if diff != "" {
+ t.Errorf("invalid key value: %v:%v", s, diff)
+ }
+ }
+
+ bytes, err := json.MarshalIndent(got, "", "")
  if err != nil {
- t.Fatal(err)
+ t.Fatalf("can't marshal json results: %v", err)
  }
- defer resp.Body.Close()
 
- resp, err = client.Get("/export/do-work")
+ t.Logf("%v", string(bytes))
+ // TODO: verify signature
+}
+
+func getKeysFromLatestBatch(t *testing.T, exportDir string, ctx context.Context, env *serverenv.ServerEnv) *export.TemporaryExposureKeyExport {
+ readmeBlob, err := env.Blobstore().GetObject(ctx, exportDir, "index.txt")
+ if err != nil {
+ t.Fatalf("Can't file index.txt in blobstore: %v", err)
+ }
+
+ exportFile := getLatestFile(readmeBlob)
+ if exportFile == "" {
+ t.Fatalf("Can't find export files in blobstore: %v", exportDir)
+ }
+
+ t.Logf("Reading keys data from: %v", exportFile)
+
+ blob, err := env.Blobstore().GetObject(ctx, exportDir, exportFile)
  if err != nil {
  t.Fatal(err)
  }
- defer resp.Body.Close()
 
- // TODO: verify export has the correct file
- // b, err := env.Blobstore().GetObject(ctx, "my-bucket", "index.txt")
- // if err != nil {
- // t.Fatal(err)
- // }
- // _ = b
+ keyExport, err := exportapi.UnmarshalExportFile(blob)
+ if err != nil {
+ t.Fatalf("can't extract export data: %v", err)
+ }
+
+ return keyExport
+}
 
- // TODO: verify signature
+func getLatestFile(indexBlob []byte) string {
+ files := strings.Split(string(indexBlob), "\n")
+
+ latestFileName := ""
+ for _, fileName := range files {
+ if strings.HasSuffix(fileName, "zip") {
+ if latestFileName == "" {
+ latestFileName = fileName
+ } else {
+ if fileName > latestFileName {
+ latestFileName = fileName
+ }
+ }
+ }
+ }
+
+ return latestFileName
 }