Fixed endpoint redirection for local broker (#909)

.gencode_hash.txt

@@ -3,7 +3,7 @@
 c70a7137ee1a30b35c9e19ca9bb54916247f9eb2e50eab37591cd2ca10b5eddb gencode/docs/config.html
 22bfb3fd7c0e73dac4cbd830aa7983714917ec6da813f6bf5768ce3aa9786fb2 gencode/docs/config_mapping.html
 1ce30b7c554a0be09a9124707cb5602dce78753304cca17c06268ecb3337da51 gencode/docs/configuration_endpoint.html
-0d78506590fa4b3371a388e168786c91574a993ac47cb9e254e70136d68416ff gencode/docs/configuration_execution.html
+73a2cfe5658e89e3a32249a235fb7c7c7232a0e1823fe8d354cb10c1b53f924b gencode/docs/configuration_execution.html
 6a097f1f87ab7b9a82e4d3aa6f6cedf69499a95742863a7f53bb5fdf53fd6ff0 gencode/docs/configuration_pod.html
 647229fdf5967f27019766244005e863fb52155d697235edeba9a84e5b6e6607 gencode/docs/configuration_pubber.html
 f24d15e549f1b143b50d6011f2f7fd03286e1cd128a3e25591195b25f8efc472 gencode/docs/data_template.html
@@ -63,7 +63,7 @@ fbe412e8c874139d828021a034cf6a6a6e1d35d01ccffe98ae35ac48187de753 gencode/java/u
 dc25e685886e11a741418be9191a478e13c0244647b5a0cac65d6c1e55055578 gencode/java/udmi/schema/Entry.java
 c92438f417b28b0b1856751f55344aaebfdfe14802d90e2b4d1b4e987462709b gencode/java/udmi/schema/Envelope.java
 9c2e8b3984e873cac3a801d94ef1ef3ecd32f64402a51a0f3292127b99b372d9 gencode/java/udmi/schema/Events.java
-684aa5160bdb62e94b414e2426d65c685b179609e1e6d63220f395d85623f045 gencode/java/udmi/schema/ExecutionConfiguration.java
+0331c395b64a2bfaa1ca7f65bf966bb7e15febda8f3391b15145c6683bab8494 gencode/java/udmi/schema/ExecutionConfiguration.java
 25815abe54d4671858f55bd5d911292bd50f122b80878064b014afcfcbd86ea7 gencode/java/udmi/schema/FamilyDiscovery.java
 2ecee68c28ee55cce63c21ff32b27d866accdd12c55f7fe894af854c5f56bb63 gencode/java/udmi/schema/FamilyDiscoveryConfig.java
 5f50abca612a3ef122f76e07590ccd9f674698ebef63e4901d49a62a0382c957 gencode/java/udmi/schema/FamilyDiscoveryModel.java
@@ -157,7 +157,7 @@ ac3facbd96f7cb2f7e387e7497d6a36af379a2687329571f250c5670f9933244 gencode/python
 b15c649657d0fb2a102a3e1ff9653c9002715d3b412d44c544a080c37b6a9811 gencode/python/udmi/schema/config_system_testing.py
 9bc72e152352ca5a966f203211fb3e87672e2e6975ba266c332e113f20ce0e3d gencode/python/udmi/schema/config_udmi.py
 cd72d3ebc324e2db939e4161a9df246adaeb2dabecc8e9c4305ebbc5984bf2a4 gencode/python/udmi/schema/configuration_endpoint.py
-ce2ff5646f2707cad62426b54f769eac763308e411ef1925aa54eae1274a8e75 gencode/python/udmi/schema/configuration_execution.py
+40fc04efd05df7ae4ee236d640914eca22a43e2b43a4076108b6ca8b6d5f0692 gencode/python/udmi/schema/configuration_execution.py
 8f65402ddb43815f4482b1c0c965d44c390dbe1561899b3d9df572458ceb97cc gencode/python/udmi/schema/configuration_pod.py
 c61ee66daa7e632850bc2705370baa8c057e7a34792c6a29b9e4ca00c0eb195b gencode/python/udmi/schema/configuration_pod_base.py
 ef61eea743cc2629893b18411636672cdfec0e209e58eb7918b33b43edab5196 gencode/python/udmi/schema/configuration_pod_bridge.py

.github/workflows/testing.yml

@@ -275,10 +275,15 @@ jobs:
  run: bin/test_udmis
  - name: mosquitto debug
  if: ${{ !cancelled() }}
- run: sudo cat /etc/mosquitto/dynamic_security.json
+ run: |
+ mosquitto -h || true
+ sudo more /etc/mosquitto/* /etc/mosquitto/conf.d/* || true
  - name: udmis log
  if: ${{ !cancelled() }}
  run: cat out/udmis*.log
+ - name: mosquitto logs
+ if: ${{ !cancelled() }}
+ run: cat /var/log/mosquitto/mosquitto.log || true
  - name: support bundle
  if: ${{ !cancelled() }}
  run: bin/support ${{ github.repository_owner }}_${{ github.job }}_

bin/mosquctl_log

@@ -0,0 +1,22 @@
+#!/bin/bash -ex
+
+UDMI_ROOT=$(dirname $0)/..
+cd $UDMI_ROOT
+
+LOG_FILE=/var/log/mosquitto/mosquitto.log
+
+source $UDMI_ROOT/etc/shell_common.sh
+
+if [[ $# != 1 ]]; then
+ echo Usage: $0 client_id_prefix
+ false
+fi
+
+prefix=$1
+shift
+
+echo $(date +%s): Starting MONITOR Client $prefix of $LOG_FILE
+
+tail -f $LOG_FILE | stdbuf -oL egrep " (from|to|as|Client) $prefix"
+
+echo $(date +%s): Finished MONITOR Client $prefix of $LOG_FILE

bin/start_local

@@ -40,6 +40,9 @@ if [[ -n ${UDMI_ALT_REGISTRY:-} ]]; then
  bin/mosquctl_site $site_model $UDMI_ALT_REGISTRY
 fi
 
+# Global access to logs for connection tracking by UDMIS.
+sudo chmod a+r /var/log/mosquitto/mosquitto.log
+
 bin/start_udmis
 
 echo Done with local server setup.

bin/test_redirect

@@ -51,7 +51,7 @@ log Running pubber with redirectRegistry on timeout
 timeout 1m unbuffer bin/pubber $site_path $project_spec $device_id $serial_no redirectRegistry=missing > $PUBBER_LOG.1 2>&1 || true
 
 log Checks that a redirect happened and failed...
-egrep -q 'registries/(.*~)?ZZ-TRI-FECTA(_.*)?/devices' $PUBBER_LOG.1 || bin/fail no ZZ-TRI-FECTA registry reference
+egrep -q '/r(egistries)?/(.*~)?ZZ-TRI-FECTA(_.*)?/d(evices)?/' $PUBBER_LOG.1 || bin/fail no ZZ-TRI-FECTA registry reference
 fgrep -q registries/missing/devices $PUBBER_LOG.1 || bin/fail no missing registry reference
 fgrep -q 'While waiting for connection start: Connection error' $PUBBER_LOG.1 || bin/fail no connection error
 fgrep -q 'Endpoint connection restored to last working endpoint' $PUBBER_LOG.1 || bin/fail no restore log message
@@ -146,7 +146,7 @@ instances=$(ps ax | fgrep pubber | fgrep java | awk '{print $1}' | wc -l)
 [[ $instances -eq 0 ]] || fail Unexpected lingering pubber instances found.
 
 log Check basic redirect-by-config setup...
-egrep -q 'registries/(.*~)?ZZ-TRI-FECTA(_.*)?/devices' $PUBBER_LOG.2 || bin/fail no ZZ-TRI-FECTA registry reference
+egrep -q '/r(egistries)?/(.*~)?ZZ-TRI-FECTA(_.*)?/d(evices)?/' $PUBBER_LOG.2 || bin/fail no ZZ-TRI-FECTA registry reference
 fgrep -q 'registries/reconfigure/devices' $PUBBER_LOG.2 || bin/fail no reconfigure registry reference
 fgrep -q 'While waiting for connection start: Connection error' $PUBBER_LOG.2 || bin/fail no connection error
 fgrep -q 'Endpoint connection restored to last working endpoint' $PUBBER_LOG.2 || bin/fail no connection restore

bin/test_regclean

@@ -148,6 +148,16 @@ jq '.iot_provider = "dynamic"' $iot_config | sponge $iot_config
 jq '.project_id = "this-is-not-right"' $iot_config | sponge $iot_config
 jq '.udmi_namespace = "bunny"' $iot_config | sponge $iot_config
 
+echo Create one device to test that it works...
+echo bin/registrar $site_args $device_id
+bin/registrar $site_args $device_id
+
+echo Check $device_id to make sure that it works...
+echo pubber/bin/run $pubber_config
+result=0
+timeout 20s pubber/bin/run $pubber_config || result=$?
+[[ $result == 0 ]] || fail Improper pubber failure code $result
+
 echo Clean out the registry to make sure devices get removed...
 echo bin/registrar $site_args -d -x
 bin/registrar $site_args -d -x

common/src/main/java/com/google/udmi/util/GeneralUtils.java

@@ -31,6 +31,9 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -636,4 +639,17 @@ public static String removeArg(List<String> argList, String description) {
  }
  return argList.remove(0);
  }
+
+ public static byte[] getFileBytes(String dataFile) {
+ Path dataPath = Paths.get(dataFile);
+ try {
+ return Files.readAllBytes(dataPath);
+ } catch (Exception e) {
+ throw new RuntimeException("While getting data from " + dataPath.toAbsolutePath(), e);
+ }
+ }
+
+ public static byte[] getFileBytes(File dataFile) {
+ return getFileBytes(dataFile.getPath());
+ }
 }

common/src/main/java/com/google/udmi/util/SiteModel.java

@@ -2,19 +2,22 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
+import static com.google.udmi.util.Common.DEFAULT_REGION;
 import static com.google.udmi.util.Common.NO_SITE;
 import static com.google.udmi.util.Common.SITE_METADATA_KEY;
 import static com.google.udmi.util.Common.getNamespacePrefix;
 import static com.google.udmi.util.GeneralUtils.OBJECT_MAPPER_RAW;
+import static com.google.udmi.util.GeneralUtils.catchToNull;
 import static com.google.udmi.util.GeneralUtils.friendlyStackTrace;
+import static com.google.udmi.util.GeneralUtils.getFileBytes;
 import static com.google.udmi.util.GeneralUtils.ifNotNullGet;
 import static com.google.udmi.util.GeneralUtils.ifNotNullThen;
 import static com.google.udmi.util.GeneralUtils.ifNullThen;
 import static com.google.udmi.util.GeneralUtils.removeArg;
+import static com.google.udmi.util.GeneralUtils.sha256;
 import static com.google.udmi.util.JsonUtil.asMap;
 import static com.google.udmi.util.JsonUtil.convertTo;
 import static com.google.udmi.util.JsonUtil.convertToStrict;
-import static com.google.udmi.util.JsonUtil.loadFile;
 import static com.google.udmi.util.JsonUtil.loadFileRequired;
 import static com.google.udmi.util.JsonUtil.loadFileStrict;
 import static com.google.udmi.util.MessageUpgrader.METADATA_SCHEMA;
@@ -83,8 +86,9 @@ public class SiteModel {
  .enable(SerializationFeature.INDENT_OUTPUT)
  .setDateFormat(new ISO8601DateFormat())
  .setSerializationInclusion(JsonInclude.Include.NON_NULL);
- private static final Pattern ID_PATTERN = Pattern.compile(
+ private static final Pattern IOT_CORE_PATTERN = Pattern.compile(
  "projects/(.*)/locations/(.*)/registries/(.*)/devices/(.*)");
+ private static final Pattern MQTT_PATTERN = Pattern.compile("/r/(.*)/d/(.*)");
  private static final String EXTRAS_DIR = "extras";
  private static final String CLOUD_IOT_CONFIG_JSON = "cloud_iot_config.json";
  private static final Pattern SPEC_PATTERN = Pattern.compile(
@@ -93,6 +97,8 @@ public class SiteModel {
  private static final int SPEC_PROJECT_GROUP = 3;
  private static final int SPEC_NAMESPACE_GROUP = 5;
  private static final File CONFIG_OUT_DIR = new File("out/");
+ private static final String RSA_PRIVATE_KEY = "rsa_private.pkcs8";
+ private static final String EC_PRIVATE_KEY = "ec_private.pkcs8";
 
  private final String sitePath;
  private final Map<String, Object> siteDefaults;
@@ -218,17 +224,29 @@ public static ClientInfo parseClientId(String clientId) {
  if (clientId == null) {
  throw new IllegalArgumentException("client_id not specified");
  }
- Matcher matcher = ID_PATTERN.matcher(clientId);
- if (!matcher.matches()) {
- throw new IllegalArgumentException(
- format("client_id %s does not match pattern %s", clientId, ID_PATTERN.pattern()));
+
+ Matcher iotCoreMatcher = IOT_CORE_PATTERN.matcher(clientId);
+ if (iotCoreMatcher.matches()) {
+ ClientInfo clientInfo = new ClientInfo();
+ clientInfo.iotProject = iotCoreMatcher.group(1);
+ clientInfo.cloudRegion = iotCoreMatcher.group(2);
+ clientInfo.registryId = iotCoreMatcher.group(3);
+ clientInfo.deviceId = iotCoreMatcher.group(4);
+ return clientInfo;
+ }
+
+ Matcher mqttMatcher = MQTT_PATTERN.matcher(clientId);
+ if (mqttMatcher.matches()) {
+ ClientInfo clientInfo = new ClientInfo();
+ clientInfo.iotProject = LOCALHOST_HOSTNAME;
+ clientInfo.cloudRegion = DEFAULT_REGION;
+ clientInfo.registryId = mqttMatcher.group(1);
+ clientInfo.deviceId = mqttMatcher.group(2);
+ return clientInfo;
  }
- ClientInfo clientInfo = new ClientInfo();
- clientInfo.iotProject = matcher.group(1);
- clientInfo.cloudRegion = matcher.group(2);
- clientInfo.registryId = matcher.group(3);
- clientInfo.deviceId = matcher.group(4);
- return clientInfo;
+
+ throw new IllegalArgumentException(format("client_id %s does not match pattern %s or %s",
+ clientId, IOT_CORE_PATTERN.pattern(), MQTT_PATTERN.pattern()));
  }
 
  public static List<String> listDevices(File devicesDir) {
@@ -542,6 +560,15 @@ public File getReflectorDir() {
  return getSubdirectory(REFLECTOR_DIR);
  }
 
+ public String getDevicePassword(String deviceId) {
+ String gatewayId = catchToNull(() -> getMetadata(deviceId).gateway.gateway_id);
+ String targetId = ofNullable(gatewayId).orElse(deviceId);
+ File rsaKeyFile = getDeviceFile(targetId, RSA_PRIVATE_KEY);
+ File ecKeyFile = getDeviceFile(targetId, EC_PRIVATE_KEY);
+ File keyFile = ecKeyFile.exists() ? ecKeyFile : rsaKeyFile;
+ return sha256(getFileBytes(keyFile)).substring(0, 8);
+ }
+
  public static class MetadataException extends Metadata {
 
  public final File file;

etc/mosquitto_udmi.conf

@@ -1,3 +1,5 @@
+log_type all
+
 listener 8883
 allow_anonymous false
 password_file /etc/mosquitto/mosquitto.passwd