fix: Ensure the LambdaRuntimeAPI server is up before launching INIT phase

Author: gregfurman

cmd/localstack/main.go

@@ -4,13 +4,15 @@ package main
 
 import (
 	"context"
-	log "github.com/sirupsen/logrus"
-	"go.amzn.com/lambda/interop"
-	"go.amzn.com/lambda/rapidcore"
 	"os"
 	"runtime/debug"
 	"strconv"
 	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"go.amzn.com/lambda/interop"
+	"go.amzn.com/lambda/rapidcore"
 )
 
 type LsOpts struct {
@@ -188,6 +190,12 @@ func main() {
 		SetLogsEgressAPI(localStackLogsEgressApi).
 		SetTracer(tracer)
 
+	// Corresponds to the 'AWS_LAMBDA_RUNTIME_API' environment variable.
+	// We need to ensure the runtime server is up before the INIT phase,
+	// but this envar is only set after the InitHandler is called.
+	runtimeAPIAddress := "127.0.0.1:9001"
+	sandbox.SetRuntimeAPIAddress(runtimeAPIAddress)
+
 	// xray daemon
 	endpoint := "http://" + lsOpts.LocalstackIP + ":" + lsOpts.EdgePort
 	xrayConfig := initConfig(endpoint, xRayLogLevel)
@@ -225,6 +233,14 @@ func main() {
 	}
 	go RunHotReloadingListener(interopServer, lsOpts.HotReloadingPaths, fileWatcherContext, lsOpts.FileWatcherStrategy)
 
+	log.Debugf("Awaiting initialization of runtime api at %s.", runtimeAPIAddress)
+	// Fixes https://github.com/localstack/localstack/issues/12680
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	if err := waitForRuntimeAPI(ctx, runtimeAPIAddress); err != nil {
+		log.Fatalf("Lambda Runtime API server at %s did not come up in 30s, with error %s", runtimeAPIAddress, err.Error())
+	}
+	cancel()
+
 	// start runtime init. It is important to start `InitHandler` synchronously because we need to ensure the
 	// notification channels and status fields are properly initialized before `AwaitInitialized`
 	log.Debugln("Starting runtime init.")
@@ -239,6 +255,7 @@ func main() {
 		return
 	}
 
+	// ping the runtime server until ready
 	log.Debugln("Completed initialization of runtime init. Sending status ready to LocalStack.")
 	if err := interopServer.localStackAdapter.SendStatus(Ready, []byte{}); err != nil {
 		log.Fatalln("Failed to send status ready to LocalStack " + err.Error() + ". Exiting.")

cmd/localstack/runtime.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+)
+
+func waitForRuntimeAPI(ctx context.Context, targetAddress string) error {
+	if !strings.HasPrefix(targetAddress, "http://") {
+		targetAddress = fmt.Sprintf("http://%s", targetAddress)
+	}
+
+	healthEndpoint, err := url.JoinPath(targetAddress, "2018-06-01", "ping")
+	if err != nil {
+		return err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthEndpoint, nil)
+	if err != nil {
+		return err
+	}
+	client := &http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+
+	for {
+		resp, err := client.Do(req)
+		if err == nil {
+			defer resp.Body.Close()
+			if resp.StatusCode == http.StatusOK {
+				return nil
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+}