From 18f3748d0e9c29812b885be9bd0b8c84940bea52 Mon Sep 17 00:00:00 2001 From: Jonathan Cobb Date: Mon, 6 Jul 2020 19:29:53 -0400 Subject: [PATCH] use timeout instead of die for ip address assignment timeout --- .../src/main/java/bubble/model/cloud/BubbleNode.java | 5 +++-- .../java/bubble/service/cloud/StandardNetworkService.java | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bubble-server/src/main/java/bubble/model/cloud/BubbleNode.java b/bubble-server/src/main/java/bubble/model/cloud/BubbleNode.java index b89f32c2..299bbda7 100644 --- a/bubble-server/src/main/java/bubble/model/cloud/BubbleNode.java +++ b/bubble-server/src/main/java/bubble/model/cloud/BubbleNode.java @@ -31,6 +31,7 @@ import javax.persistence.*; import java.io.File; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import static bubble.ApiConstants.EP_NODES; import static bubble.model.cloud.BubbleNodeState.*; @@ -246,11 +247,11 @@ public class BubbleNode extends IdentifiableBase implements HasNetwork, HasBubbl return new BubbleNodeQuickClient(this, configuration); } - public void waitForIpAddresses() { + public void waitForIpAddresses() throws TimeoutException { final long start = now(); while ((!hasIp4() || !hasIp6()) && now() - start < IP_ADDR_TIMEOUT) { sleep(TimeUnit.SECONDS.toMillis(2), "waiting for node to have IP addresses"); } - if (!hasIp4() || !hasIp6()) die("waitForIpAddresses: timeout"); + if (!hasIp4() || !hasIp6()) throw new TimeoutException("waitForIpAddresses: timeout"); } } diff --git a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java index e0a8ca09..3e4682e5 100644 --- a/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java +++ b/bubble-server/src/main/java/bubble/service/cloud/StandardNetworkService.java @@ -65,6 +65,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import static bubble.ApiConstants.*; @@ -252,7 +253,11 @@ public class StandardNetworkService implements NetworkService { // Prepare ansible roles // We must wait until after server is started, because some roles require ip4 in vars - node.waitForIpAddresses(); + try { + node.waitForIpAddresses(); + } catch (TimeoutException e) { + launchFailureCanRetry(node, e, "waitForIpAddresses error"); + } progressMeter.write(METER_TICK_PREPARING_ROLES); final Map ctx = ansiblePrep.prepAnsible( automation, bubbleFilesDir, account, network, node, computeDriver,