ci(*:skip) Update e2e tests directory structure

.github/workflows/e2e.yml

@@ -61,44 +61,55 @@ jobs:
  matrix:
  include:
  - directory: bare
+ subdir: baree2e
 
  - directory: bare-https
+ subdir: httpse2e
 
  - directory: bare-client-auth
+ subdir: clientauthe2e
 
- - directory: framework-jax
+ - directory: jax
+ subdir: jaxe2e
 
- - directory: framework-pytorch
+ - directory: pytorch
+ subdir: pytorche2e
  dataset: |
  from torchvision.datasets import CIFAR10
  CIFAR10('./data', download=True)
 
- - directory: framework-tensorflow
+ - directory: tensorflow
+ subdir: tfe2e
  dataset: |
  import tensorflow as tf
  tf.keras.datasets.cifar10.load_data()
 
- - directory: framework-opacus
+ - directory: opacus
+ subdir: opacuse2e
  dataset: |
  from torchvision.datasets import CIFAR10
  CIFAR10('./data', download=True)
 
- - directory: framework-pytorch-lightning
+ - directory: pytorch-lightning
+ subdir: lightninge2e
  dataset: |
  from torchvision.datasets import MNIST
  MNIST('./data', download=True)
 
- - directory: framework-scikit-learn
+ - directory: scikit-learn
+ subdir: scikite2e
  dataset: |
  import openml
  openml.datasets.get_dataset(554)
 
- - directory: framework-fastai
+ - directory: fastai
+ subdir: fastaie2e
  dataset: |
  from fastai.vision.all import untar_data, URLs
  untar_data(URLs.MNIST)
 
- - directory: framework-pandas
+ - directory: pandas
+ subdir: pandase2e
  dataset: |
  from pathlib import Path
  from sklearn.datasets import load_iris
@@ -135,12 +146,15 @@ jobs:
  if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
  run: |
  python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+ - name: Install subdir components
+ run: pip install .
  - name: Download dataset
  if: ${{ matrix.dataset }}
  run: python -c "${{ matrix.dataset }}"
  - name: Run edge client test
  if: ${{ matrix.directory != 'bare-client-auth' }}
- run: ./../test_legacy.sh "${{ matrix.directory }}"
+ working-directory: e2e/${{ matrix.directory }}/${{ matrix.subdir }}
+ run: ./../../test_legacy.sh "${{ matrix.directory }}"
  - name: Run virtual client test
  if: ${{ matrix.directory != 'bare-client-auth' }}
  run: python simulation.py

e2e/bare-client-auth/clientauthe2e/__init__.py

@@ -0,0 +1 @@
+"""clientatuthe2e."""

e2e/bare-client-auth/pyproject.toml

@@ -21,8 +21,8 @@ allow-direct-references = true
 publisher = "flwrlabs"
 
 [tool.flwr.app.components]
-serverapp = ""
-clientapp = "client:app"
+serverapp = "clientauthe2e.server_app:app"
+clientapp = "clientauthe2e.client_app:app"
 
 [tool.flwr.app.config]
 

e2e/bare-https/httpse2e/__init__.py

@@ -0,0 +1 @@
+"""httpse2e."""

e2e/bare-https/client.py → e2e/bare-https/httpse2e/client_app.py

@@ -39,6 +39,6 @@ def client_fn(context: Context):
  start_client(
  server_address="127.0.0.1:8080",
  client=FlowerClient().to_client(),
- root_certificates=Path("certificates/ca.crt").read_bytes(),
+ root_certificates=Path("../certificates/ca.crt").read_bytes(),
  insecure=False,
  )

e2e/bare-https/httpse2e/server_app.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+
+import flwr as fl
+
+app = fl.server.ServerApp()
+
+
+@app.main()
+def main(driver, context):
+ # Construct the LegacyContext
+ context = fl.server.LegacyContext(
+ context=context,
+ config=fl.server.ServerConfig(num_rounds=3),
+ )
+
+ # Create the workflow
+ workflow = fl.server.workflow.DefaultWorkflow()
+
+ # Execute
+ workflow(driver, context)
+
+ hist = context.history
+ assert (
+ hist.losses_distributed[-1][1] == 0
+ or (hist.losses_distributed[0][1] / hist.losses_distributed[-1][1]) >= 0.98
+ )
+
+
+if __name__ == "__main__":
+ hist = fl.server.start_server(
+ server_address="127.0.0.1:8080",
+ config=fl.server.ServerConfig(num_rounds=3),
+ certificates=(
+ Path("../certificates/ca.crt").read_bytes(),
+ Path("../certificates/server.pem").read_bytes(),
+ Path("../certificates/server.key").read_bytes(),
+ ),
+ )
+
+ assert (
+ hist.losses_distributed[-1][1] == 0
+ or (hist.losses_distributed[0][1] / hist.losses_distributed[-1][1]) >= 0.98
+ )

e2e/bare-https/pyproject.toml

@@ -21,8 +21,8 @@ allow-direct-references = true
 publisher = "flwrlabs"
 
 [tool.flwr.app.components]
-serverapp = "server:app"
-clientapp = "client:app"
+serverapp = "httpse2e.server_app:app"
+clientapp = "httpse2e.client_app:app"
 
 [tool.flwr.app.config]
 

e2e/bare/baree2e/__init__.py

@@ -0,0 +1 @@
+"""baree2e."""

e2e/bare/pyproject.toml

@@ -21,8 +21,8 @@ allow-direct-references = true
 publisher = "flwrlabs"
 
 [tool.flwr.app.components]
-serverapp = ""
-clientapp = "client:app"
+serverapp = "baree2e.server_app:app"
+clientapp = "baree2e.client_app:app"
 
 [tool.flwr.app.config]
 

e2e/bare/simulation.py

@@ -1,7 +1,7 @@
 from typing import List, Tuple
 
 import numpy as np
-from client import client_fn
+from baree2e.client_app import client_fn
 
 import flwr as fl
 from flwr.common import Metrics

e2e/fastai/fastaie2e/__init__.py

@@ -0,0 +1 @@
+"""fastaie2e."""

e2e/fastai/fastaie2e/server_app.py

@@ -0,0 +1,79 @@
+import numpy as np
+
+import flwr as fl
+
+STATE_VAR = "timestamp"
+
+
+# Define metric aggregation function
+def record_state_metrics(metrics):
+ """Ensure that timestamps are monotonically increasing."""
+ if not metrics:
+ return {}
+
+ if STATE_VAR not in metrics[0][1]:
+ # Do nothing if keyword is not present
+ return {}
+
+ states = []
+ for _, m in metrics:
+ # split string and covert timestamps to float
+ states.append([float(tt) for tt in m[STATE_VAR].split(",")])
+
+ for client_state in states:
+ if len(client_state) == 1:
+ continue
+ deltas = np.diff(client_state)
+ assert np.all(
+ deltas > 0
+ ), f"Timestamps are not monotonically increasing: {client_state}"
+
+ return {STATE_VAR: states}
+
+
+app = fl.server.ServerApp()
+
+
+@app.main()
+def main(driver, context):
+ # Construct the LegacyContext
+ context = fl.server.LegacyContext(
+ context=context,
+ config=fl.server.ServerConfig(num_rounds=3),
+ )
+
+ # Create the workflow
+ workflow = fl.server.workflow.DefaultWorkflow()
+
+ # Execute
+ workflow(driver, context)
+
+ hist = context.history
+ assert (
+ hist.losses_distributed[-1][1] == 0
+ or (hist.losses_distributed[0][1] / hist.losses_distributed[-1][1]) >= 0.98
+ )
+
+
+if __name__ == "__main__":
+ strategy = fl.server.strategy.FedAvg(
+ evaluate_metrics_aggregation_fn=record_state_metrics
+ )
+
+ hist = fl.server.start_server(
+ server_address="0.0.0.0:8080",
+ config=fl.server.ServerConfig(num_rounds=3),
+ strategy=strategy,
+ )
+
+ assert (
+ hist.losses_distributed[-1][1] == 0
+ or (hist.losses_distributed[0][1] / hist.losses_distributed[-1][1]) >= 0.98
+ )
+
+ if STATE_VAR in hist.metrics_distributed:
+ # The checks in record_state_metrics don't do anythinng if client's state has a single entry
+ state_metrics_last_round = hist.metrics_distributed[STATE_VAR][-1]
+ assert (
+ len(state_metrics_last_round[1][0]) == 2 * state_metrics_last_round[0]
+ ), "There should be twice as many entries in the client state as rounds"

e2e/framework-fastai/pyproject.toml → e2e/fastai/pyproject.toml

@@ -23,8 +23,8 @@ allow-direct-references = true
 publisher = "flwrlabs"
 
 [tool.flwr.app.components]
-serverapp = ""
-clientapp = "client:app"
+serverapp = "fastaie2e.server_app:app"
+clientapp = "fastaie2e.client_app:app"
 
 [tool.flwr.app.config]
 

e2e/framework-fastai/simulation.py → e2e/fastai/simulation.py

@@ -1,4 +1,4 @@
-from client import client_fn
+from fastaie2e.client_app import client_fn
 
 import flwr as fl
 

e2e/jax/jaxe2e/__init__.py

@@ -0,0 +1 @@
+"""jaxe2e."""

e2e/framework-jax/client.py → e2e/jax/jaxe2e/client_app.py

@@ -3,8 +3,8 @@
 from typing import Dict, List, Tuple
 
 import jax
-import jax_training
 import numpy as np
+from jaxe2e import jax_training
 
 from flwr.client import ClientApp, NumPyClient, start_client
 from flwr.common import Context