feat: parallel

automated_test.py

@@ -19,9 +19,10 @@ def gt_c2f(gt):
  f_gt[ f_gt == mx ] = 3
  return f_gt
 
+@pytest.mark.parametrize("parallel", (1,2))
 @pytest.mark.parametrize("connectivity", (4, 6, 8, 18, 26))
 @pytest.mark.parametrize("dtype", TEST_TYPES)
-def test_2d_square(dtype, connectivity):
+def test_2d_square(dtype, connectivity, parallel):
  def test(order, ground_truth):
  input_labels = np.zeros( (16,16), dtype=dtype, order=order )
  input_labels[:8,:8] = 8
@@ -30,7 +31,7 @@ def test(order, ground_truth):
  input_labels[8:,8:] = 11
 
  output_labels = cc3d.connected_components(
- input_labels, connectivity=connectivity
+ input_labels, connectivity=connectivity, parallel=parallel
  ).astype(dtype)
 
  print(output_labels)
@@ -229,13 +230,16 @@ def test(order, ground_truth):
  test("C", ground_truth)
  test("F", gt_c2f(ground_truth))
 
+@pytest.mark.parametrize("parallel", (1,2))
 @pytest.mark.parametrize("order", ('C', 'F'))
 @pytest.mark.parametrize("connectivity", (6,18,26))
-def test_3d_all_different(order, connectivity):
+def test_3d_all_different(order, connectivity, parallel):
  input_labels = np.arange(0, 100 * 99 * 98).astype(np.uint32) + 1
  input_labels = input_labels.reshape((100,99,98), order=order)
 
- output_labels = cc3d.connected_components(input_labels, connectivity=connectivity)
+ output_labels = cc3d.connected_components(
+ input_labels, connectivity=connectivity, parallel=parallel
+ )
 
  assert np.unique(output_labels).shape[0] == 100*99*98
  assert output_labels.shape == (100, 99, 98)
@@ -442,7 +446,8 @@ def test_max_labels_nonsensical():
  assert np.all(real_labels == zero_labels)
  assert np.all(real_labels == negative_labels)
 
-def test_compare_scipy_26():
+@pytest.mark.parametrize("parallel", (1,2))
+def test_compare_scipy_26(parallel):
  import scipy.ndimage.measurements
 
  sx, sy, sz = 128, 128, 128
@@ -454,7 +459,9 @@ def test_compare_scipy_26():
  [[1,1,1], [1,1,1], [1,1,1]]
  ]
 
- cc3d_labels, Ncc3d = cc3d.connected_components(labels, connectivity=26, return_N=True)
+ cc3d_labels, Ncc3d = cc3d.connected_components(
+ labels, connectivity=26, return_N=True, parallel=parallel
+ )
  scipy_labels, Nscipy = scipy.ndimage.measurements.label(labels, structure=structure)
 
  print(cc3d_labels)

benchmarks/benchmarks.py

@@ -5,6 +5,35 @@
 import fastremap
 import time
 
+def prettyprint(tm, voxels):
+ print("{:.2f} sec. {:.2f} MVx/sec".format(tm, voxels / tm / (1000*1000)))
+
+def summary(times, voxels):
+ times = times[1:]
+ mean = sum(times) / float(len(times))
+
+ print("mean:")
+ prettyprint(mean, voxels)
+ print("stddev: %.2f" % np.std(times))
+
+structures = {
+ 6: [
+ [[0,0,0], [0,1,0], [0,0,0]],
+ [[0,1,0], [1,1,1], [0,1,0]],
+ [[0,0,0], [0,1,0], [0,0,0]]
+ ],
+ 18: [
+ [[0,1,0], [1,1,1], [0,1,0]],
+ [[1,1,1], [1,1,1], [1,1,1]],
+ [[0,1,0], [1,1,1], [0,1,0]]
+ ],
+ 26: [
+ [[1,1,1], [1,1,1], [1,1,1]],
+ [[1,1,1], [1,1,1], [1,1,1]],
+ [[1,1,1], [1,1,1], [1,1,1]]
+ ]
+}
+
 def cc3d_mutlilabel_extraction(labels):
  res, N = cc3d.connected_components(labels, return_N=True)
  for label, extracted in tqdm(cc3d.each(labels, binary=True, in_place=True), total=N):
@@ -26,40 +55,37 @@ def ndimage_mutlilabel_extraction(labels):
  for ccid in tqdm(range(1,N+1)):
  extracted = (res == ccid)
 
-def pinky40subvol():
- from cloudvolume import CloudVolume
- return CloudVolume('gs://neuroglancer/wms/skeletonization/pinky40subvol', 
- mip=2, progress=True, cache=True)[:,:,:][...,0]
+def test_binary_image_processing(labels, connectivity=26):
+ voxels = labels.size
+ times = []
 
+ for label, img in tqdm(cc3d.each(labels, binary=True, in_place=True)):
+ s = time.time()
+ cc3d.connected_components(img)
+ # scipy.ndimage.measurements.label(labels, structure=structures[connectivity])
+ e = time.time()
+ dt = e-s
+ times.append(dt)
+ prettyprint(dt, voxels)
+ summary(times, voxels)
 
+def test_multilabel_speed(labels, connectivity=26):
+ voxels = labels.size
+ times = []
 
-labels = pinky40subvol()
-cc3d_mutlilabel_extraction(labels)
-# ndimage_mutlilabel_extraction(labels)
+ for i in range(10):
+ s = time.time()
+ cc3d.connected_components(labels, connectivity=connectivity)
+ e = time.time()
+ dt = e-s
+ times.append(dt)
+ prettyprint(dt, voxels)
+ summary(times, voxels)
 
-# voxels = labels.size
-# times = []
-# # for label, img in tqdm(cc3d.each(labels, binary=False, in_place=True)):
-# # s = time.time()
-# # cc3d.connected_components(img)
-# # # scipy.ndimage.measurements.label(labels, structure=structure)
-# # e = time.time()
-# # dt = e-s
-# # times.append(dt)
-# # prettyprint(dt, voxels)
+labels = np.load("connectomics.npy")
+# cc3d_mutlilabel_extraction(labels)
+# ndimage_mutlilabel_extraction(labels)
 
-# for i in range(10):
-# s = time.time()
-# cc3d.connected_components(labels)#, connectivity=26)
-# # scipy.ndimage.measurements.label(labels)
-# e = time.time()
-# dt = e-s
-# times.append(dt)
-# prettyprint(dt, voxels)
+test_multilabel_speed(labels)
 
-# times = times[1:]
-# mean = sum(times) / float(len(times))
 
-# print("mean:")
-# prettyprint(mean, voxels)
-# print("stddev: %.2f" % np.std(times))