Move profile to /tmp and add CLOUD_QUERY_LOOPS (#888)

faucetsdn · May 14, 2024 · 84f293e · 84f293e
1 parent 8508642
commit 84f293e
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 5 deletions.
diff --git a/udmis/bin/run b/udmis/bin/run
@@ -11,7 +11,7 @@ shift
 ROOT=$(dirname $0)/..
 cd $ROOT
 
-PROFILE_BASE=profile/
+PROFILE_BASE=/tmp/profile/
 mkdir -p $PROFILE_BASE
 
 echo

diff --git a/udmis/etc/k8s_config.yaml b/udmis/etc/k8s_config.yaml
@@ -6,4 +6,5 @@ data:
  BITBOX_DISCOVERY:
  ETCD_CLUSTER:
  PROFILE_SEC: "60"
+ CLOUD_QUERY_LOOPS:
  UDMI_NAMESPACE: @UDMI_NAMESPACE@
diff --git a/udmis/etc/k8s_udmis.yaml b/udmis/etc/k8s_udmis.yaml
@@ -46,6 +46,12 @@ spec:
  name: config
  key: ETCD_CLUSTER
  optional: true
+ - name: CLOUD_QUERY_LOOPS
+ valueFrom:
+ configMapKeyRef:
+ name: config
+ key: CLOUD_QUERY_LOOPS
+ optional: true
  - name: PROFILE_SEC
  valueFrom:
  configMapKeyRef:

diff --git a/udmis/etc/peringknife~k8s_config.yaml b/udmis/etc/peringknife~k8s_config.yaml
@@ -6,4 +6,5 @@ data:
  BITBOX_DISCOVERY: prodev~udmi_target-discover
  ETCD_CLUSTER: etcd-set-0.etcd-set
  PROFILE_SEC: "10"
+ CLOUD_QUERY_LOOPS: "3"
  UDMI_NAMESPACE: @UDMI_NAMESPACE@
diff --git a/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java b/udmis/src/main/java/com/google/bos/udmi/service/access/ClearBladeIotAccessProvider.java
@@ -333,6 +333,7 @@ private HashMap<String, CloudModel> fetchDevices(String deviceRegistryId, String
  RegistryName.of(projectId, location, deviceRegistryId).getRegistryFullName();
  String pageToken = null;
  HashMap<String, CloudModel> collect = new HashMap<>();
+ int queryCount = 0;
  do {
  DevicesListRequest request = DevicesListRequest.Builder.newBuilder().setParent(
  registryFullName)
@@ -346,8 +347,9 @@ private HashMap<String, CloudModel> fetchDevices(String deviceRegistryId, String
  .collect(Collectors.toMap(Entry::getKey, Entry::getValue));
  collect.putAll(responseMap);
  pageToken = response.getNextPageToken();
- debug(format("fetchDevices %s found %d total %d more %s", deviceRegistryId,
- responseMap.size(), collect.size(), pageToken != null));
+ queryCount++;
+ debug(format("fetchDevices %s #%d found %d total %d more %s", deviceRegistryId,
+ queryCount, responseMap.size(), collect.size(), pageToken != null));
  } while (pageToken != null);
  return collect;
  }

diff --git a/udmis/src/main/java/com/google/bos/udmi/service/core/CloudQueryHandler.java b/udmis/src/main/java/com/google/bos/udmi/service/core/CloudQueryHandler.java
@@ -8,8 +8,10 @@
 import static com.google.udmi.util.GeneralUtils.toDate;
 import static com.google.udmi.util.JsonUtil.stringifyTerse;
 import static java.util.Objects.requireNonNull;
+import static java.util.Optional.ofNullable;
 
 import com.google.bos.udmi.service.access.IotAccessBase;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -35,6 +37,8 @@ public class CloudQueryHandler {
  private final Envelope envelope;
  private final String savedEnvelope;
  private final String savedQuery;
+ private final int fetchCount =
+ Integer.parseInt(ofNullable(System.getenv("CLOUD_QUERY_LOOPS")).orElse("1"));
 
  /**
  * Create a query handler for cloud queries.
@@ -131,12 +135,19 @@ private void queryRegistryDevices() {
  String deviceRegistryId = requireNonNull(envelope.deviceRegistryId, "registry id");
  requireNull(envelope.deviceId, "device id");
 
- CloudModel cloudModel = iotAccess.listDevices(deviceRegistryId);
+ // TODO: Remove hacky workaround once ClearBlade API is known to return consistent results.
+ Set<Entry<String, CloudModel>> deviceSet = new HashSet<>();
+ for (int loop = 1; loop <= fetchCount; loop++) {
+ CloudModel cloudModel = iotAccess.listDevices(deviceRegistryId);
+ deviceSet.addAll(cloudModel.device_ids.entrySet());
+ debug("Queried registry %s #%d for %d totaling %d",
+ envelope.deviceRegistryId, loop, cloudModel.device_ids.size(), deviceSet.size());
+ }
 
  DiscoveryEvents discoveryEvent = new DiscoveryEvents();
  discoveryEvent.scan_family = ProtocolFamily.IOT;
  discoveryEvent.generation = query.generation;
- discoveryEvent.devices = cloudModel.device_ids.entrySet().stream().collect(Collectors.toMap(
+ discoveryEvent.devices = deviceSet.stream().collect(Collectors.toMap(
  Entry::getKey, entry -> convertDeviceEntry(entry.getValue())));
  publish(discoveryEvent);