Merge pull request #6943 from MinaProtocol/hotfix/catch-transition-fr…

…ontier-errors

hotfix/catch-transition-frontier-errors Update transition_frontier.ml…

buildkite/scripts/generate-diff.sh

@@ -2,6 +2,9 @@
 
 # Base against origin/develop by default, but use pull-request base otherwise
 BASE=${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-develop}
+TAG=$(git tag --points-at HEAD)
+
+[[ -n $TAG ]] && git ls-files && exit
 
 # Finds the commit hash of HEAD of $BASE branch
 BASECOMMIT=$(git log origin/$BASE -1 --pretty=format:%H)

dockerfiles/Dockerfile-coda-daemon

@@ -40,6 +40,7 @@ touch .coda-config/coda-prover.log\n\
 touch .coda-config/coda-verifier.log\n\
 touch .coda-config/mina-best-tip.log\n\
 while true; do\n\
+  rm -f /root/.coda-config/.mina-lock\n\
   coda "$@" 2>&1 >coda.log &\n\
   coda_pid=$!\n\
   tail -q -f coda.log -f .coda-config/coda-prover.log -f .coda-config/coda-verifier.log -f .coda-config/mina-best-tip.log &\n\

dockerfiles/Dockerfile-coda-daemon-puppeteered

@@ -12,11 +12,13 @@ pgrep -f --newest "python3 /root/coda_daemon_puppeteer.py"'\
 > find_puppeteer.sh
 
 RUN echo '#!/bin/bash\n\
-kill -s SIGUSR2 $(./find_puppeteer.sh)'\
+kill -s SIGUSR2 $(./find_puppeteer.sh)\n\
+while [ ! -f daemon-active ]; do sleep 1; done'\
 > start.sh
 
 RUN echo '#!/bin/bash\n\
-kill -s SIGUSR1 $(./find_puppeteer.sh)'\
+kill -s SIGUSR1 $(./find_puppeteer.sh)\n\
+while [ -f daemon-active ]; do sleep 1; done'\
 > stop.sh
 
 RUN chmod +x find_puppeteer.sh start.sh stop.sh

dockerfiles/coda_daemon_puppeteer.py

@@ -41,11 +41,13 @@ def start_daemon():
         stdout=f,
         stderr=subprocess.STDOUT
     )
+  Path('daemon-active').touch()
 
 def stop_daemon():
   global coda_process
   coda_process.send_signal(signal.SIGTERM)
   coda_process.wait()
+  Path('daemon-active').unlink()
   coda_process = None
 
 # technically, doing the loops like this will eventually result in a stack overflow

helm/archive-node/Chart.lock

@@ -1,9 +1,9 @@
 dependencies:
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
-  version: 10.1.0
+  version: 10.1.1
 - name: common-utilities
   repository: https://coda-charts.storage.googleapis.com
-  version: 0.0.2
-digest: sha256:db3e05bb7d5ae36ba2ac91248b1a43727aa22d2754a1a5e2d27dd7a9535148ea
-generated: "2020-11-23T15:34:07.09326598-05:00"
+  version: 0.0.3
+digest: sha256:3388409d529973a81eb5107309b782ba12a90fed96e0f14c3759a4e87c64639e
+generated: "2020-11-30T12:00:08.338922221-05:00"

helm/archive-node/Chart.yaml

@@ -2,14 +2,14 @@ apiVersion: v2
 name: archive-node
 description: A Helm chart for Mina Protocol's archive node
 type: application
-version: 0.2.1
+version: 0.2.2
 appVersion: 1.16.0
 dependencies:
   - name: postgresql
     version: "*"
     repository: "@bitnami"
   - name: common-utilities
-    version: 0.0.2
+    version: 0.0.3
     repository: "https://coda-charts.storage.googleapis.com"
 icon: https://github.com/MinaProtocol/mina/blob/develop/frontend/website/public/static/img/[email protected]
 keywords:

helm/archive-node/templates/_healthchecks.tpl

@@ -6,7 +6,7 @@ archive-node startup probe settings
 {{- define "healthcheck.archive.startupProbe" }}
 startupProbe:
   tcpSocket:
-    port: archive-postgres-port
+    port: postgres-port
 {{- end }}
 
 {{/*
@@ -15,7 +15,7 @@ archive-node liveness check settings
 {{- define "healthcheck.archive.livenessCheck" }}
 livenessProbe:
   tcpSocket:
-    port: archive-server-port
+    port: archive-port
 {{- include "healthcheck.common.settings" . | indent 2 }}
 {{- end }}
 

helm/archive-node/templates/archive.yaml

@@ -104,13 +104,14 @@ spec:
         env:
         imagePullPolicy: Always
         ports:
-        - name: archive-server-port
+        - name: archive-port
           protocol: TCP
           containerPort: {{ .Values.archive.ports.server }}
-        - name: archive-postgres-port
+        - name: postgres-port
           protocol: TCP
           containerPort: {{ .Values.archive.ports.postgres }}
 {{- include "healthcheck.archive.allChecks" .Values | indent 8 }}
+{{- include "nodeSelector.preemptible" .Values | indent 6 }}
       volumes:
       - name: config-dir
         emptyDir: {}

helm/archive-node/values.yaml

@@ -37,3 +37,6 @@ healthcheck:
   failureThreshold: 60
   periodSeconds: 5
   initialDelaySeconds: 30
+
+nodeSelector:
+  preemptible: false

helm/block-producer/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: common-utilities
   repository: https://coda-charts.storage.googleapis.com
-  version: 0.0.2
-digest: sha256:adfdc569e9aea9832a7b25bde8dfa043f9e27864ed302417eb69fd0650c0e05e
-generated: "2020-11-23T15:38:29.057989588-05:00"
+  version: 0.0.3
+digest: sha256:aed0ad114400ccbd45e8552059b01bb7054d84910a2c576e4a714f9e3d6f7b8d
+generated: "2020-11-24T12:50:11.452718343-05:00"

helm/block-producer/Chart.yaml

@@ -2,11 +2,11 @@ apiVersion: v2
 name: block-producer
 description: A Helm chart for Mina Protocol's block producing network nodes
 type: application
-version: 0.2.1
+version: 0.2.2
 appVersion: 1.16.0
 dependencies:
   - name: common-utilities
-    version: 0.0.2
+    version: 0.0.3
     repository: "https://coda-charts.storage.googleapis.com"
 icon: https://github.com/MinaProtocol/mina/blob/develop/frontend/website/public/static/img/[email protected]
 keywords:

helm/block-producer/templates/block-producer.yaml

@@ -228,6 +228,7 @@ spec:
         - name: daemon-config
           mountPath: "/config/"
         {{- end }}
+{{- include "nodeSelector.preemptible" $.Values | indent 6 }}
       volumes:
       - name: private-keys
         secret:

helm/block-producer/values.yaml

@@ -73,3 +73,6 @@ healthcheck:
   failureThreshold: 60
   periodSeconds: 5
   initialDelaySeconds: 30
+
+nodeSelector:
+  preemptible: true

helm/common/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: common-utilities
 description: A Helm chart for Mina Protocol's common chart utilities
 type: library
-version: 0.0.2
+version: 0.0.3
 appVersion: 0.0.1
 dependencies:
 icon:

helm/common/templates/_healthchecks.tpl

@@ -37,6 +37,8 @@ Daemon readiness check settings
 readinessProbe:
   exec:
     command: [
+      "/bin/bash",
+      "-c",
       "source /healthcheck/utilities.sh && isDaemonSynced"
     ]
 {{- include "healthcheck.common.settings" . | indent 2 }}

helm/common/templates/_node-selectors.tpl

@@ -0,0 +1,21 @@
+### Mina node selector TEMPLATES ###
+
+{{/*
+Node selector: preemptible node affinity
+*/}}
+{{- define "nodeSelector.preemptible" }}
+{{- if .nodeSelector.preemptible }}
+nodeSelector:
+  cloud.google.com/gke-preemptible: "true"
+{{- end }}
+{{- end }}
+
+{{/*
+Node selector: custom affinity mapping
+*/}}
+{{- define "nodeSelector.customMapping" }}
+{{- if . }}
+nodeSelector:
+{{ toYaml . | nindent 2 }}
+{{- end }}
+{{- end }}

helm/common/values.yaml

@@ -5,3 +5,5 @@ healthcheck:
   failureThreshold: 60
   periodSeconds: 5
   initialDelaySeconds: 30
+nodeSelector:
+  preemptible: false

helm/seed-node/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: common-utilities
   repository: https://coda-charts.storage.googleapis.com
-  version: 0.0.2
-digest: sha256:adfdc569e9aea9832a7b25bde8dfa043f9e27864ed302417eb69fd0650c0e05e
-generated: "2020-11-23T15:39:05.659540471-05:00"
+  version: 0.0.3
+digest: sha256:aed0ad114400ccbd45e8552059b01bb7054d84910a2c576e4a714f9e3d6f7b8d
+generated: "2020-11-24T12:51:45.744101108-05:00"

helm/seed-node/Chart.yaml

@@ -2,11 +2,11 @@ apiVersion: v2
 name: seed-node
 description: A Helm chart for Mina Protocol's seed nodes
 type: application
-version: 0.2.1
+version: 0.2.2
 appVersion: 1.16.0
 dependencies:
   - name: common-utilities
-    version: 0.0.2
+    version: 0.0.3
     repository: "https://coda-charts.storage.googleapis.com"
 icon: https://github.com/MinaProtocol/mina/blob/develop/frontend/website/public/static/img/[email protected]
 keywords:

helm/seed-node/templates/_healthchecks.tpl

@@ -21,6 +21,8 @@ seed-node readiness settings
 readinessProbe:
   exec:
     command: [
+      "/bin/bash",
+      "-c",
       "source /healthcheck/utilities.sh && isDaemonSynced && peerCountGreaterThan 0"
     ]
 {{- end }}

helm/seed-node/templates/seed-node.yaml

@@ -101,4 +101,5 @@ spec:
         configMap:
           name: {{ tpl .Values.seed.fullname . }}-daemon-config
       {{- end -}}
+{{- include "nodeSelector.preemptible" .Values | indent 6 }}
 {{- end }}

helm/seed-node/values.yaml

@@ -21,3 +21,6 @@ healthcheck:
   failureThreshold: 60
   periodSeconds: 5
   initialDelaySeconds: 30
+
+nodeSelector:
+  preemptible: false

helm/snark-worker/Chart.lock

@@ -1,6 +1,6 @@
 dependencies:
 - name: common-utilities
   repository: https://coda-charts.storage.googleapis.com
-  version: 0.0.2
-digest: sha256:adfdc569e9aea9832a7b25bde8dfa043f9e27864ed302417eb69fd0650c0e05e
-generated: "2020-11-23T15:39:31.341392464-05:00"
+  version: 0.0.3
+digest: sha256:aed0ad114400ccbd45e8552059b01bb7054d84910a2c576e4a714f9e3d6f7b8d
+generated: "2020-11-24T12:52:06.870229993-05:00"

helm/snark-worker/Chart.yaml

@@ -2,11 +2,11 @@ apiVersion: v2
 name: snark-worker
 description: A Helm chart for Mina Protocol's SNARK worker nodes
 type: application
-version: 0.2.1
+version: 0.2.2
 appVersion: 1.16.0
 dependencies:
   - name: common-utilities
-    version: 0.0.2
+    version: 0.0.3
     repository: "https://coda-charts.storage.googleapis.com"
 icon: https://github.com/MinaProtocol/mina/blob/develop/frontend/website/public/static/img/[email protected]
 keywords:

helm/snark-worker/templates/snark-coordinator.yaml

@@ -96,4 +96,5 @@ spec:
         configMap:
           name: snark-coordinator-daemon-config
       {{- end }}
+{{- include "nodeSelector.preemptible" .Values | indent 6 }}
 {{- end }}

helm/snark-worker/templates/snark-worker.yaml

@@ -37,4 +37,5 @@ spec:
           - name: "RAYON_NUM_THREADS"
             value: "4"
         imagePullPolicy: Always
+{{- include "nodeSelector.preemptible" .Values | nindent 6 }}
 {{ end }}

helm/snark-worker/values.yaml

@@ -38,3 +38,6 @@ healthcheck:
   failureThreshold: 60
   periodSeconds: 5
   initialDelaySeconds: 30
+
+nodeSelector:
+  preemptible: true

src/app/cli/src/coda.ml

@@ -934,10 +934,26 @@ Pass one of -peer, -peer-list-file, -seed.|} ;
       let external_transition_database_dir =
         conf_dir ^/ "external_transition_database"
       in
-      let%bind () = Async.Unix.mkdir ~p:() external_transition_database_dir in
-      let external_transition_database =
-        Auxiliary_database.External_transition_database.create ~logger
-          external_transition_database_dir
+      let%bind external_transition_database =
+        let create_db () =
+          let%map () =
+            Async.Unix.mkdir ~p:() external_transition_database_dir
+          in
+          Auxiliary_database.External_transition_database.create ~logger
+            external_transition_database_dir
+        in
+        match%bind Deferred.Or_error.try_with create_db with
+        | Ok res ->
+            return res
+        | Error err ->
+            [%log warn]
+              "Encountered an error $err while creating the external \
+               transition database. Retrying."
+              ~metadata:[("err", Error_json.error_to_yojson err)] ;
+            let%bind () =
+              File_system.remove_dir external_transition_database_dir
+            in
+            create_db ()
       in
       trace_database_initialization "external_transition_database" __LOC__
         external_transition_database_dir ;

src/app/libp2p_helper/src/libp2p_helper/main.go

@@ -446,12 +446,21 @@ func (s *subscribeMsg) run(app *app) (interface{}, error) {
 			return pubsub.ValidationIgnore
 		}
 
+		deadline, ok := ctx.Deadline()
+		if !ok {
+			app.P2p.Logger.Errorf("no deadline set on validation context")
+			defer app.ValidatorMutex.Unlock()
+			delete(app.Validators, seqno)
+			return pubsub.ValidationIgnore
+		}
+
 		app.writeMsg(validateUpcall{
-			Sender: sender,
-			Data:   codaEncode(msg.Data),
-			Seqno:  seqno,
-			Upcall: "validate",
-			Idx:    s.Subscription,
+			Sender:     sender,
+			Expiration: deadline.UnixNano(),
+			Data:       codaEncode(msg.Data),
+			Seqno:      seqno,
+			Upcall:     "validate",
+			Idx:        s.Subscription,
 		})
 
 		// Wait for the validation response, but be sure to honor any timeout/deadline in ctx
@@ -557,11 +566,12 @@ func (u *unsubscribeMsg) run(app *app) (interface{}, error) {
 }
 
 type validateUpcall struct {
-	Sender *codaPeerInfo `json:"sender"`
-	Data   string        `json:"data"`
-	Seqno  int           `json:"seqno"`
-	Upcall string        `json:"upcall"`
-	Idx    int           `json:"subscription_idx"`
+	Sender     *codaPeerInfo `json:"sender"`
+	Expiration int64         `json:"expiration"`
+	Data       string        `json:"data"`
+	Seqno      int           `json:"seqno"`
+	Upcall     string        `json:"upcall"`
+	Idx        int           `json:"subscription_idx"`
 }
 
 type validationCompleteMsg struct {

src/app/rosetta/README.md

@@ -4,6 +4,10 @@ Implementation of the [Rosetta API](https://www.rosetta-api.org/) for Mina.
 
 ## Changelog
 
+11/30:
+
+- Upgrades from Rosetta spec 1.4.4 to 1.4.7
+
 9/14:
 
 - Upgrades from Rosetta spec v1.4.2 to v1.4.4