Merge branch 'release-v0.14.1' into 'v0.14-stable'

Resolve "Release v0.14.1"

Release issue #925

See merge request secml/secml!31

.gitlab-ci.yml

@@ -111,7 +111,13 @@ package:docs:
 
 
 code_quality:
- interruptible: true
+ interruptible: true 
+ rules:
+ - if: '$CODE_QUALITY_DISABLED'
+ when: never
+ - if: '$CI_COMMIT_BRANCH == "stable" || $CI_COMMIT_BRANCH =~ /^release-.*$/ || $CI_COMMIT_BRANCH =~ /^.*-stable$/'
+ when: never
+ - if: '$CI_COMMIT_BRANCH'
  variables:
  REPORT_FORMAT: html
  artifacts:
@@ -562,6 +568,22 @@ release:gitlab-pages:
  - git commit -m "Release $CI_COMMIT_TAG"
  - git push
 
+release:zoo:
+ stage: release
+ variables:
+ RELEASE: $CI_COMMIT_TAG
+ PIP_CACHE_DIR: $PIP_CACHE_DIR
+ TORCH_HOME: $TORCH_HOME
+ SECML_HOME_DIR: $SECML_HOME_DIR
+ trigger:
+ project: secml/secml-zoo
+ strategy: depend
+ rules:
+ - if: '$CI_SERVER_HOST != "gitlab.com"'
+ when: never
+ - if: $CI_COMMIT_TAG
+ when: manual
+
 release:pypi:
  extends: .release
  image: ${CI_REGISTRY}/pralab/docker-helper-images/python36-setuptools:latest

CHANGELOG.md

@@ -1,3 +1,19 @@
+## v0.14.1 (22/04/2021)
+- This version brings fixes for a few issues with the optimizers and related classes, along with improvements to documentation for all attacks, optimizers, and related classes.
+
+### Fixed (3 changes)
+- #923 Fixed `COptimizerPGDLS` and `COptimizerPGDLS` not working properly if the classifier's gradient has multiple components with the same (max) value.
+- #919 Fixed `CConstraintL1` crashing when projecting sparse data using default center value (scalar 0).
+- #920 Fixed inconsistent results between dense and sparse data for `CConstraintL1` projection caused by type casting.
+
+### Removed & Deprecated (1 change)
+- #922 Removed unnecessary parameter `discrete` from `COptimizerPGDLS` and `COptimizerPGDExp`.
+
+### Documentation (2 changes)
+- #100017 Improved documentation of `CAttackEvasion`, `COptimizer`, `CLineSearch`, and corresponding subclasses.
+- #918 Installing the latest stable version of RobustBench instead of the master version.
+
+
 ## v0.14 (23/03/2021)
 - #795 Added new package `adv.attacks.evasion.foolbox` with a wrapper for [Foolbox](https://foolbox.readthedocs.io/en/stable/).
 - #623 `secml` is now tested for compatibility with Python 3.8.

src/secml/VERSION

@@ -1 +1 @@
-0.14
+0.14.1

src/secml/adv/attacks/evasion/c_attack_evasion.py

@@ -29,7 +29,7 @@ class CAttackEvasion(CAttack, metaclass=ABCMeta):
  belonging to the `y_target` class.
  attack_classes : 'all' or CArray, optional
  Array with the classes that can be manipulated by the attacker or
-  'all' (default) if all classes can be manipulated.
+ 'all' (default) if all classes can be manipulated.
 
  """
  __super__ = 'CAttackEvasion'
@@ -200,6 +200,8 @@ def run(self, x, y, ds_init=None):
 
  y_pred = CArray(y_pred)
 
+ self.logger.info("y_pred after attack:\n{:}".format(y_pred))
+
  # Return the mean objective function value on the evasion points
  f_obj = fs_opt.mean()
 

src/secml/adv/attacks/evasion/c_attack_evasion_pgd.py

@@ -7,7 +7,6 @@
 .. moduleauthor:: Marco Melis <[email protected]>
 
 """
-from secml import _NoValue
 from secml.adv.attacks.evasion import CAttackEvasionPGDLS
 
 
@@ -41,8 +40,8 @@ class CAttackEvasionPGD(CAttackEvasionPGDLS):
  double_init_ds : CDataset or None, optional
  Dataset used to initialize an alternative init point (double init).
  double_init : bool, optional
-  If True (default), use double initialization point.
-  Needs double_init_ds not to be None.
+ If True (default), use double initialization point.
+ Needs double_init_ds not to be None.
  distance : {'l1' or 'l2'}, optional
  Norm to use for computing the distance of the adversarial example
  from the original sample. Default 'l2'.
@@ -58,10 +57,11 @@ class CAttackEvasionPGD(CAttackEvasionPGDLS):
  belonging to the `y_target` class.
  attack_classes : 'all' or CArray, optional
  Array with the classes that can be manipulated by the attacker or
-  'all' (default) if all classes can be manipulated.
+ 'all' (default) if all classes can be manipulated.
  solver_params : dict or None, optional
- Parameters for the solver. Default None, meaning that default
- parameters will be used.
+ Parameters for the solver.
+ Default None, meaning that default parameters will be used.
+ See :class:`COptimizerPGD` for more information.
 
  Attributes
  ----------
@@ -77,7 +77,6 @@ def __init__(self, classifier,
  dmax=0,
  lb=0,
  ub=1,
- discrete=_NoValue,
  y_target=None,
  attack_classes='all',
  solver_params=None):
@@ -91,10 +90,6 @@ def __init__(self, classifier,
  # class (indiscriminate evasion). See _get_point_with_min_f_obj()
  self._xk = None
 
- # pgd solver does not accepts parameter `discrete`
- if discrete is not _NoValue:
- raise ValueError("`pgd` solver does not work in discrete space.")
-
  super(CAttackEvasionPGD, self).__init__(
  classifier=classifier,
  double_init_ds=double_init_ds,

src/secml/adv/attacks/evasion/c_attack_evasion_pgd_exp.py

@@ -38,8 +38,8 @@ class onto the feasible domain and try again.
  double_init_ds : CDataset or None, optional
  Dataset used to initialize an alternative init point (double init).
  double_init : bool, optional
-  If True (default), use double initialization point.
-  Needs double_init_ds not to be None.
+ If True (default), use double initialization point.
+ Needs double_init_ds not to be None.
  distance : {'l1' or 'l2'}, optional
  Norm to use for computing the distance of the adversarial example
  from the original sample. Default 'l2'.
@@ -55,10 +55,11 @@ class onto the feasible domain and try again.
  belonging to the `y_target` class.
  attack_classes : 'all' or CArray, optional
  Array with the classes that can be manipulated by the attacker or
-  'all' (default) if all classes can be manipulated.
+ 'all' (default) if all classes can be manipulated.
  solver_params : dict or None, optional
- Parameters for the solver. Default None, meaning that default
- parameters will be used.
+ Parameters for the solver.
+ Default None, meaning that default parameters will be used.
+ See :class:`COptimizerPGDExp` for more information.
 
  Attributes
  ----------

src/secml/adv/attacks/evasion/c_attack_evasion_pgd_ls.py

@@ -49,8 +49,8 @@ class CAttackEvasionPGDLS(CAttackEvasion, CAttackMixin):
  double_init_ds : CDataset or None, optional
  Dataset used to initialize an alternative init point (double init).
  double_init : bool, optional
-  If True (default), use double initialization point.
-  Needs double_init_ds not to be None.
+ If True (default), use double initialization point.
+ Needs double_init_ds not to be None.
  distance : {'l1' or 'l2'}, optional
  Norm to use for computing the distance of the adversarial example
  from the original sample. Default 'l2'.
@@ -66,10 +66,11 @@ class CAttackEvasionPGDLS(CAttackEvasion, CAttackMixin):
  belonging to the `y_target` class.
  attack_classes : 'all' or CArray, optional
  Array with the classes that can be manipulated by the attacker or
-  'all' (default) if all classes can be manipulated.
+ 'all' (default) if all classes can be manipulated.
  solver_params : dict or None, optional
- Parameters for the solver. Default None, meaning that default
- parameters will be used.
+ Parameters for the solver.
+ Default None, meaning that default parameters will be used.
+ See :class:`COptimizerPGDLS` for more information.
 
  Attributes
  ----------

src/secml/adv/attacks/evasion/tests/c_attack_evasion_testcases.py

@@ -1,7 +1,9 @@
 from secml.testing import CUnitTest
 
-from numpy import *
+import os
+import numpy as np
 
+from secml.array import CArray
 from secml.data.loader import CDLRandomBlobs
 from secml.optim.constraints import \
  CConstraintBox, CConstraintL1, CConstraintL2
@@ -19,7 +21,7 @@
 class CAttackEvasionTestCases(CUnitTest):
  """Unittests interface for CAttackEvasion."""
  images_folder = IMAGES_FOLDER
- make_figures = False # Set as True to produce figures
+ make_figures = os.getenv('MAKE_FIGURES', False) # True to produce figures
 
  def _load_blobs(self, n_feats, n_clusters, sparse=False, seed=None):
  """Load Random Blobs dataset.
@@ -71,6 +73,11 @@ def _discretize_data(ds, eta):
  else: # eta is a single value
  ds.X = (ds.X / eta).round() * eta
 
+ # It is likely that after the discretization there are duplicates
+ new_array = [tuple(row) for row in ds.X.tondarray()]
+ uniques, uniques_idx = np.unique(new_array, axis=0, return_index=True)
+ ds = ds[uniques_idx.tolist(), :]
+
  return ds
 
  def _prepare_linear_svm(self, sparse, seed):
@@ -102,6 +109,35 @@ def _prepare_linear_svm(self, sparse, seed):
 
  return ds, clf
 
+ def _prepare_linear_svm_10d(self, sparse, seed):
+ """Preparare the data required for attacking a LINEAR SVM.
+
+ - load a blob 10D dataset
+ - create a SVM (C=1) and a minmax preprocessor
+
+ Parameters
+ ----------
+ sparse : bool
+ seed : int or None
+
+ Returns
+ -------
+ ds : CDataset
+ clf : CClassifierSVM
+
+ """
+ ds = self._load_blobs(
+ n_feats=10, # Number of dataset features
+ n_clusters=2, # Number of dataset clusters
+ sparse=sparse,
+ seed=seed
+ )
+
+ normalizer = CNormalizerMinMax(feature_range=(-1, 1))
+ clf = CClassifierSVM(C=1.0, preprocess=normalizer)
+
+ return ds, clf
+
  def _prepare_nonlinear_svm(self, sparse, seed):
  """Preparare the data required for attacking a NONLINEAR SVM.
 

src/secml/adv/attacks/evasion/tests/test_c_attack_evasion_pgd_exp.py

@@ -5,8 +5,8 @@
 from secml.array import CArray
 
 
-class TestCAttackEvasionPGDLS(CAttackEvasionTestCases):
- """Unittests for CAttackEvasionPGDLS."""
+class TestCAttackEvasionPGDExp(CAttackEvasionTestCases):
+ """Unittests for CAttackEvasionPGDExp."""
 
  def _set_evasion(self, ds, params):
  """Prepare the evasion attack.
@@ -79,6 +79,93 @@ def test_linear_l1(self):
 
  self._plot_2d_evasion(evas, ds, x0, 'pgd_exp_linear_L1.pdf')
 
+ def test_linear_l1_discrete(self):
+ """Test evasion of a linear classifier using L1 distance (discrete)."""
+
+ eta = 0.5
+ sparse = True
+ seed = 10
+
+ ds, clf = self._prepare_linear_svm(sparse, seed)
+
+ ds = self._discretize_data(ds, eta)
+
+ evasion_params = {
+ "classifier": clf,
+ "double_init_ds": ds,
+ "distance": 'l1',
+ "dmax": 2,
+ "lb": -1,
+ "ub": 1,
+ "attack_classes": CArray([1]),
+ "y_target": 0,
+ "solver_params": {
+ "eta": eta,
+ "eta_min": None,
+ "eta_max": None
+ }
+ }
+
+ evas, x0, y0 = self._set_evasion(ds, evasion_params)
+
+ # Expected final optimal point
+ expected_x = CArray([0.5, -1])
+ expected_y = 0
+
+ self._run_evasion(evas, x0, y0, expected_x, expected_y)
+
+ self._plot_2d_evasion(evas, ds, x0, 'pgd_exp_linear_L1_discrete.pdf')
+
+ def test_linear_l1_discrete_10d(self):
+ """Test evasion of a linear classifier (10 features)
+ using L1 distance (discrete).
+ In this test we set few features to the same value to cover a
+ special case of the l1 projection, where there are multiple
+ features with the same max value. The optimizer should change
+ one of them at each iteration.
+ """
+
+ eta = 0.5
+ sparse = True
+ seed = 10
+
+ ds, clf = self._prepare_linear_svm_10d(sparse, seed)
+
+ ds = self._discretize_data(ds, eta)
+
+ evasion_params = {
+ "classifier": clf,
+ "double_init_ds": ds,
+ "distance": 'l1',
+ "dmax": 5,
+ "lb": -2,
+ "ub": 2,
+ "attack_classes": CArray([1]),
+ "y_target": 0,
+ "solver_params": {
+ "eta": eta,
+ "eta_min": None,
+ "eta_max": None
+ }
+ }
+
+ evas, x0, y0 = self._set_evasion(ds, evasion_params)
+
+ # Set few features to the same max value
+ w_new = clf.w.deepcopy()
+ w_new[CArray.randint(
+ clf.w.size, shape=3, random_state=seed)] = clf.w.max()
+ clf._w = w_new
+
+ # Expected final optimal point
+ # CAttackEvasionPGDExp uses CLineSearchBisectProj
+ # which brings the point outside of the grid
+ expected_x = \
+ CArray([-1.8333, -1.8333, 1.8333, 0, -0.5, 0, 0.5, -0.5, 1, 0.5])
+ expected_y = 0
+
+ self._run_evasion(evas, x0, y0, expected_x, expected_y)
+
  def test_linear_l2(self):
  """Test evasion of a linear classifier using L2 distance."""