Add configurable jstack profiling (#877)

.gitignore

@@ -23,6 +23,7 @@ credentials.json
 /udmis/build/
 /udmis/out/
 /udmis/var/
+/udmis/profile/
 /udmis/.idea/libraries/
 /selfie/build/
 /pubber/build/

udmis/bin/run

@@ -11,6 +11,9 @@ shift
 ROOT=$(dirname $0)/..
 cd $ROOT
 
+PROFILE_BASE=profile/
+mkdir -p $PROFILE_BASE
+
 echo
 echo Initial env:
 env
@@ -78,5 +81,24 @@ echo Final java env:
 env
 echo
 
+JSTACK=$(dirname $(realpath `which java`))/jstack
+if [[ -n $PROFILE_SEC ]]; then
+ # BASHPID will be replaced with the java process becasue of the exec below
+ TPID=$BASHPID
+
+ echo Profiling java process $TPID usage every ${PROFILE_SEC}s
+ (
+ cd $PROFILE_BASE
+ while [[ -d /proc/$TPID ]]; do
+ sleep $PROFILE_SEC
+ outfile=stack_$(printf %08d $SECONDS).txt
+ echo Writing stack profile to $outfile
+ $JSTACK $TPID > $outfile
+ stale_files=$(ls -rt | head -n -1000)
+ [[ -n $stale_files ]] && rm $stale_files
+ done
+ ) &
+fi
+
 # Run with exec to replace shell so java receives SIGTERM signal.
 exec java -jar $POD_JAR $POD_CONFIG

udmis/etc/k8s_config.yaml

@@ -5,4 +5,5 @@ metadata:
 data:
  BITBOX_DISCOVERY:
  ETCD_CLUSTER:
+ PROFILE_SEC: "60"
  UDMI_NAMESPACE: @UDMI_NAMESPACE@

udmis/etc/k8s_udmis.yaml

@@ -46,6 +46,12 @@ spec:
  name: config
  key: ETCD_CLUSTER
  optional: true
+ - name: PROFILE_SEC
+ valueFrom:
+ configMapKeyRef:
+ name: config
+ key: PROFILE_SEC
+ optional: true
  - name: BITBOX_DISCOVERY
  valueFrom:
  configMapKeyRef:

udmis/etc/peringknife~k8s_config.yaml

@@ -5,4 +5,5 @@ metadata:
 data:
  BITBOX_DISCOVERY: prodev~udmi_target-discover
  ETCD_CLUSTER: etcd-set-0.etcd-set
+ PROFILE_SEC: "10"
  UDMI_NAMESPACE: @UDMI_NAMESPACE@