Early key release (google#706)

* Allow for v1.5+ early key release. Multiple keys can be provided that all have the same start interval
* Add configuration params for this
* Add tests for new pieces. 100% coverage on exposure model transform.
* Add documentation.
* add same day release as an optional feature to the generate server

Fixes google#705
Part of google#663

docs/getting-started/publishing-temporary-exposure-keys.md

@@ -19,7 +19,42 @@ exposed, their keys are shared to the server in order for applications to
 download and [determine if other users interacted with any of the now exposed
 keys](https://blog.google/documents/69/Exposure_Notification_-_Cryptography_Specification_v1.2.1.pdf).
 
-## Publishing Keys
+## The Publish API Call
+
+TEKs are published by sending the appropriate JSON document in the body of
+an HTTP POST request to the `exposure` server.
+
+The structure of the API is defined in [pkg/api/v1alpha1/exposure_types.go](https:/google/exposure-notifications-server/blob/main/pkg/api/v1alpha1/exposure_types.go),
+in the `Publish` type. Please see the documentation in the source file
+for details of the fields themselves.
+
+Here, we point out some non-obvious validation that is applied to the keys.
+
+* All keys must be valid! If there are any validation errors, the entire batch
+ is rejected.
+* Max keys per publish: Default is `20` and can be adjusted with the
+ `MAX_KEYS_ON_PUBLISH` environment variable.
+* Max overlapping keys with same start interval: Default is `3` and can be
+ adjusted with the `MAX_SAME_START_INTERVAL_KEYS` environment variable.
+ In practical terms, this means that if you are obtaining TEK history on a
+ mobile device with >= v1.5 of the device API, it will stop the validity
+ of the current day's TEK and issue a new now. Both keys will have the same
+ start iterval.
+* Max age: How old keys can be. The default is `360h` (15 days) and can be
+ adjusted with the `MAX_INTERVAL_AGE_ON_PUBLISH`. All provided keys must have
+ a `rollingStartNumber` that is >= to the max age.
+* Keys with a future start time (`rollingStartNumber` indicates time > now),
+ are rejected.
+* Keys that are "sill valid" are accepted by the server, but they are embargoed
+ until after they key could no longer be replayed usefully. A stall valid key
+ is one where the `rollingStartNumber` is in the past, but the
+ `rollingStartNumber` + the `rollingPeriod` indicates a future time.
+* When using health authority verification certificates
+ (__strongly recommended__), the TEK data in the publish request and the
+ `hmackey` must be able to be used to calculate the HMAC value as present in
+ the certificate.
+
+## Server Access Configuration
 
 In order for your application to publish keys to the server, the server
 requires the registration of the Application Name (for Android) or the Bundle ID

internal/generate/config.go

@@ -36,13 +36,15 @@ type Config struct {
  SecretManager secrets.Config
  ObservabilityExporter observability.Config
 
- Port string `env:"PORT, default=8080"`
- NumExposures int `env:"NUM_EXPOSURES_GENERATED, default=10"`
- KeysPerExposure int `env:"KEYS_PER_EXPOSURE, default=14"`
- MaxKeysOnPublish int `env:"MAX_KEYS_ON_PUBLISH, default=15"`
- MaxIntervalAge time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
- TruncateWindow time.Duration `env:"TRUNCATE_WINDOW, default=1h"`
- DefaultRegion string `env:"DEFAULT_REGOIN, default=US"`
+ Port string `env:"PORT, default=8080"`
+ NumExposures int `env:"NUM_EXPOSURES_GENERATED, default=10"`
+ KeysPerExposure int `env:"KEYS_PER_EXPOSURE, default=14"`
+ MaxKeysOnPublish int `env:"MAX_KEYS_ON_PUBLISH, default=15"`
+ MaxSameStartIntervalKeys int `env:"MAX_SAME_START_INTERVAL_KEYS, default=2"`
+ SimulateSameDayRelease bool `env:"SIMULATE_SAME_DAY_RELEASE, default=false"`
+ MaxIntervalAge time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
+ TruncateWindow time.Duration `env:"TRUNCATE_WINDOW, default=1h"`
+ DefaultRegion string `env:"DEFAULT_REGOIN, default=US"`
 }
 
 func (c *Config) DatabaseConfig() *database.Config {

internal/generate/handler.go

@@ -18,6 +18,7 @@ import (
  "context"
  "fmt"
  "net/http"
+ "sort"
  "strings"
  "time"
 
@@ -38,7 +39,7 @@ func NewHandler(ctx context.Context, config *Config, env *serverenv.ServerEnv) (
  return nil, fmt.Errorf("missing database in server environment")
  }
 
- transformer, err := model.NewTransformer(config.MaxKeysOnPublish, config.MaxIntervalAge, config.TruncateWindow, false)
+ transformer, err := model.NewTransformer(config.MaxKeysOnPublish, config.MaxSameStartIntervalKeys, config.MaxIntervalAge, config.TruncateWindow, false)
  if err != nil {
  return nil, fmt.Errorf("model.NewTransformer: %w", err)
  }
@@ -86,6 +87,19 @@ func (h *generateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
  Regions: regions,
  AppPackageName: "generated.data",
  }
+ if h.config.SimulateSameDayRelease {
+ sort.Slice(publish.Keys, func(i int, j int) bool {
+ return publish.Keys[i].IntervalNumber < publish.Keys[j].IntervalNumber
+ })
+ lastKey := &publish.Keys[len(publish.Keys)-1]
+ newLastDayKey, err := util.RandomExposureKey(lastKey.IntervalNumber, 144, lastKey.TransmissionRisk)
+ if err != nil {
+ logger.Errorf("unable to simulate same day key release: %v", err)
+ } else {
+ lastKey.IntervalCount = lastKey.IntervalCount / 2
+ publish.Keys = append(publish.Keys, newLastDayKey)
+ }
+ }
 
  exposures, err := h.transformer.TransformPublish(ctx, &publish, batchTime)
  if err != nil {

internal/integration/integration.go

@@ -151,10 +151,11 @@ func testServer(tb testing.TB) (*serverenv.ServerEnv, *http.Client) {
 
  // Publish
  publishConfig := &publish.Config{
- MaxKeysOnPublish: 15,
- MaxIntervalAge: 360 * time.Hour,
- TruncateWindow: 1 * time.Second,
- DebugReleaseSameDayKeys: true,
+ MaxKeysOnPublish: 15,
+ MaxSameStartIntervalKeys: 2,
+ MaxIntervalAge: 360 * time.Hour,
+ TruncateWindow: 1 * time.Second,
+ DebugReleaseSameDayKeys: true,
  }
 
  publishHandler, err := publish.NewHandler(ctx, publishConfig, env)

internal/publish/config.go

@@ -41,12 +41,16 @@ type Config struct {
  Verification verification.Config
  ObservabilityExporter observability.Config
 
- Port string `env:"PORT, default=8080"`
- MaxKeysOnPublish int `env:"MAX_KEYS_ON_PUBLISH, default=15"`
- MaxIntervalAge time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
- TruncateWindow time.Duration `env:"TRUNCATE_WINDOW, default=1h"`
+ Port string `env:"PORT, default=8080"`
+ MaxKeysOnPublish int `env:"MAX_KEYS_ON_PUBLISH, default=20"`
+ // Provides compatibility w/ 1.5 release.
+ MaxSameStartIntervalKeys int `env:"MAX_SAME_START_INTERVAL_KEYS, default=3"`
+ MaxIntervalAge time.Duration `env:"MAX_INTERVAL_AGE_ON_PUBLISH, default=360h"`
+ TruncateWindow time.Duration `env:"TRUNCATE_WINDOW, default=1h"`
 
- // Flags for local development and testing.
+ // Flags for local development and testing. This will cause still valid keys
+ // to not be embargoed.
+ // Normallly "still valid" keys can be accepted, but are embargoed.
  DebugReleaseSameDayKeys bool `env:"DEBUG_RELEASE_SAME_DAY_KEYS"`
 }
 

internal/publish/model/exposure_model.go

@@ -103,7 +103,8 @@ func TimeForIntervalNumber(interval int32) time.Time {
 
 // Transformer represents a configured Publish -> Exposure[] transformer.
 type Transformer struct {
- maxExposureKeys int
+ maxExposureKeys int // Overall maximum number of keys.
+ maxSameDayKeys int // Number of keys that are allowed to have the same start interval.
  maxIntervalStartAge time.Duration // How many intervals old does this server accept?
  truncateWindow time.Duration
  debugReleaseSameDay bool // If true, still valid keys are not embargoed.
@@ -112,18 +113,23 @@ type Transformer struct {
 // NewTransformer creates a transformer for turning publish API requests into
 // records for insertion into the database. On the call to TransformPublish
 // all data is validated according to the transformer that is used.
-func NewTransformer(maxExposureKeys int, maxIntervalStartAge time.Duration, truncateWindow time.Duration, releaseSameDayKeys bool) (*Transformer, error) {
+func NewTransformer(maxExposureKeys int, maxSameDayKeys int, maxIntervalStartAge time.Duration, truncateWindow time.Duration, releaseSameDayKeys bool) (*Transformer, error) {
  if maxExposureKeys <= 0 {
  return nil, fmt.Errorf("maxExposureKeys must be > 0, got %v", maxExposureKeys)
  }
+ if maxSameDayKeys < 1 {
+ return nil, fmt.Errorf("maxSameDayKeys must be >= 1, got %v", maxSameDayKeys)
+ }
  return &Transformer{
  maxExposureKeys: maxExposureKeys,
+ maxSameDayKeys: maxSameDayKeys,
  maxIntervalStartAge: maxIntervalStartAge,
  truncateWindow: truncateWindow,
  debugReleaseSameDay: releaseSameDayKeys,
  }, nil
 }
 
+// KeyTransform represents the settings to apply when transforming an individual key on a publish request.
 type KeyTransform struct {
  MinStartInterval int32
  MaxStartInterval int32
@@ -165,7 +171,7 @@ func TransformExposureKey(exposureKey verifyapi.ExposureKey, appPackageName stri
  createdAt := settings.CreatedAt
  // If the key is valid beyond the current interval number. Adjust the createdAt time for the key.
  if exposureKey.IntervalNumber+exposureKey.IntervalCount > settings.MaxStartInterval {
- // key is still valid. The created At for this key needs to be adjusted unless debuggin is enabled.
+ // key is still valid. The created At for this key needs to be adjusted unless debugging is enabled.
  if !settings.ReleaseStillValidKeys {
  createdAt = TimeForIntervalNumber(exposureKey.IntervalNumber + exposureKey.IntervalCount).Truncate(settings.BatchWindow)
  }
@@ -238,25 +244,51 @@ func (t *Transformer) TransformPublish(ctx context.Context, inData *verifyapi.Pu
  entities = append(entities, exposure)
  }
 
- // Ensure that the uploaded keys are for a consecutive time period. No
- // overlaps and no gaps.
- // 1) Sort by interval number.
+ // Validate the uploaded data meets configuration parameters.
+ // In v1.5+, it is possible to have multiple keys that overlap. They
+ // take the form of the same start interval with variable rolling period numbers.
+ // Sort by interval number to make necessary checks easier.
  sort.Slice(entities, func(i int, j int) bool {
+ if entities[i].IntervalNumber == entities[j].IntervalNumber {
+ return entities[i].IntervalCount == entities[j].IntervalCount
+ }
  return entities[i].IntervalNumber < entities[j].IntervalNumber
  })
- // 2) Walk the slice and verify no gaps/overlaps.
- // We know the slice isn't empty, seed w/ the first interval.
- nextInterval := entities[0].IntervalNumber
+ // Check that any overlapping keys meet configuration.
+ // Overlapping keys must have the same start interval. And there is a max number
+ // of "same day" keys that are allowed.
+ // We do not enforce that keys have UTC midnight aligned start intervals.
+
+ // Running count of start intervals.
+ startIntervals := make(map[int32]int)
+ lastInterval := entities[0].IntervalNumber
+ nextInterval := entities[0].IntervalNumber + entities[0].IntervalCount
+
  for _, ex := range entities {
+ // Relies on the default value of 0 for the map value type.
+ startIntervals[ex.IntervalNumber] = startIntervals[ex.IntervalNumber] + 1
+
+ if ex.IntervalNumber == lastInterval {
+ // OK, overlaps by start interval. But move out the nextInterval
+ nextInterval = ex.IntervalNumber + ex.IntervalCount
+ continue
+ }
+
  if ex.IntervalNumber < nextInterval {
- if t.debugReleaseSameDay {
- logging.FromContext(ctx).Errorf("exposure keys have overlapping intervals")
- break
- }
- return nil, fmt.Errorf("exposure keys have overlapping intervals")
+ msg := fmt.Sprintf("exposure keys have non aligned overlapping intervals. %v overlaps with previous key that is good from %v to %v.", ex.IntervalNumber, lastInterval, nextInterval)
+ logging.FromContext(ctx).Errorf(msg)
+ return nil, fmt.Errorf(msg)
  }
+ // OK, current key starts at or after the end of the previous one. Advance both variables.
+ lastInterval = ex.IntervalNumber
  nextInterval = ex.IntervalNumber + ex.IntervalCount
  }
 
+ for k, v := range startIntervals {
+ if v > t.maxSameDayKeys {
+ return nil, fmt.Errorf("too many overlapping keys for start interval: %v want: <= %v, got: %v", k, t.maxSameDayKeys, v)
+ }
+ }
+
  return entities, nil
 }