caddyhttp: Honor grace period in background (#5043)

* caddyhttp: Honor grace period in background

This avoids blocking during config reloads.

* Don't quit process until servers shut down

* Make tests more likely to pass on fast CI (#5045)

* caddyhttp: Even faster shutdowns

Simultaneously shut down all HTTP servers, rather than one at a time.

In practice there usually won't be more than 1 that lingers. But this
code ensures that they all Shutdown() in their own goroutine
and then we wait for them at the end (if exiting).

We also wait for them to start up so we can be fairly confident the
shutdowns have begun; i.e. old servers no longer
accepting new connections.

* Fix comment typo

* Pull functions out of loop, for readability

7 files changed, 18 insertions, 1 deletions

diff --git a/caddytest/caddytest.go b/caddytest/caddytest.go
index 4fb3394..e560ab4 100644
--- a/caddytest/caddytest.go
+++ b/caddytest/caddytest.go
@@ -43,7 +43,7 @@ type Defaults struct {
 
 // Default testing values
 var Default = Defaults{
-	AdminPort:          2019,
+	AdminPort:          2999, // different from what a real server also running on a developer's machine might be
 	Certifcates:        []string{"/caddy.localhost.crt", "/caddy.localhost.key"},
 	TestRequestTimeout: 5 * time.Second,
 	LoadRequestTimeout: 5 * time.Second,
diff --git a/caddytest/integration/caddyfile_test.go b/caddytest/integration/caddyfile_test.go
index 2758883..77ac54e 100644
--- a/caddytest/integration/caddyfile_test.go
+++ b/caddytest/integration/caddyfile_test.go
@@ -16,6 +16,7 @@ func TestRespond(t *testing.T) {
   {
     http_port     9080
     https_port    9443
+    grace_period  1
   }
   
   localhost:9080 {
@@ -37,6 +38,7 @@ func TestRedirect(t *testing.T) {
   {
     http_port     9080
     https_port    9443
+    grace_period  1
   }
   
   localhost:9080 {
@@ -86,6 +88,7 @@ func TestReadCookie(t *testing.T) {
   {
     http_port     9080
     https_port    9443
+    grace_period  1
   }
   
   localhost:9080 {
@@ -109,6 +112,7 @@ func TestReplIndex(t *testing.T) {
   {
     http_port     9080
     https_port    9443
+    grace_period  1
   }
 
   localhost:9080 {
diff --git a/caddytest/integration/handler_test.go b/caddytest/integration/handler_test.go
index fb986c7..e295dc5 100644
--- a/caddytest/integration/handler_test.go
+++ b/caddytest/integration/handler_test.go
@@ -13,6 +13,7 @@ func TestBrowse(t *testing.T) {
 	{
 		http_port     9080
 		https_port    9443
+		grace_period  1
 	}
 	http://localhost:9080 {
 		file_server browse
diff --git a/caddytest/integration/map_test.go b/caddytest/integration/map_test.go
index afed0c3..0595957 100644
--- a/caddytest/integration/map_test.go
+++ b/caddytest/integration/map_test.go
@@ -13,6 +13,7 @@ func TestMap(t *testing.T) {
 	tester.InitServer(`{
 		http_port     9080
 		https_port    9443
+		grace_period  1
 	}
 
 	localhost:9080 {
diff --git a/caddytest/integration/reverseproxy_test.go b/caddytest/integration/reverseproxy_test.go
index ca11287..dfc8df0 100644
--- a/caddytest/integration/reverseproxy_test.go
+++ b/caddytest/integration/reverseproxy_test.go
@@ -18,6 +18,7 @@ func TestSRVReverseProxy(t *testing.T) {
 	{
 		"apps": {
 		  "http": {
+			"grace_period": 1,
 			"servers": {
 			  "srv0": {
 				"listen": [
@@ -50,6 +51,7 @@ func TestSRVWithDial(t *testing.T) {
 	{
 		"apps": {
 		  "http": {
+			"grace_period": 1,
 			"servers": {
 			  "srv0": {
 				"listen": [
@@ -115,6 +117,7 @@ func TestDialWithPlaceholderUnix(t *testing.T) {
 	{
 		"apps": {
 		  "http": {
+			"grace_period": 1,
 			"servers": {
 			  "srv0": {
 				"listen": [
@@ -156,6 +159,7 @@ func TestReverseProxyWithPlaceholderDialAddress(t *testing.T) {
 	{
 		"apps": {
 			"http": {
+				"grace_period": 1,
 				"servers": {
 					"srv0": {
 						"listen": [
@@ -239,6 +243,7 @@ func TestReverseProxyWithPlaceholderTCPDialAddress(t *testing.T) {
 	{
 		"apps": {
 			"http": {
+				"grace_period": 1,
 				"servers": {
 					"srv0": {
 						"listen": [
@@ -321,6 +326,7 @@ func TestSRVWithActiveHealthcheck(t *testing.T) {
 	{
 		"apps": {
 		  "http": {
+			"grace_period": 1,
 			"servers": {
 			  "srv0": {
 				"listen": [
diff --git a/caddytest/integration/sni_test.go b/caddytest/integration/sni_test.go
index 813e9a0..aa47c75 100644
--- a/caddytest/integration/sni_test.go
+++ b/caddytest/integration/sni_test.go
@@ -15,6 +15,7 @@ func TestDefaultSNI(t *testing.T) {
       "http": {
         "http_port": 9080,
         "https_port": 9443,
+        "grace_period": 1,
         "servers": {
           "srv0": {
             "listen": [
@@ -112,6 +113,7 @@ func TestDefaultSNIWithNamedHostAndExplicitIP(t *testing.T) {
       "http": {
         "http_port": 9080,
         "https_port": 9443,
+        "grace_period": 1,
         "servers": {
           "srv0": {
             "listen": [
@@ -212,6 +214,7 @@ func TestDefaultSNIWithPortMappingOnly(t *testing.T) {
       "http": {
         "http_port": 9080,
         "https_port": 9443,
+        "grace_period": 1,
         "servers": {
           "srv0": {
             "listen": [
diff --git a/caddytest/integration/stream_test.go b/caddytest/integration/stream_test.go
index cfd9d36..0cb1db2 100644
--- a/caddytest/integration/stream_test.go
+++ b/caddytest/integration/stream_test.go
@@ -27,6 +27,7 @@ func TestH2ToH2CStream(t *testing.T) {
       "http": {
         "http_port": 9080,
         "https_port": 9443,
+		"grace_period": 1,
         "servers": {
           "srv0": {
             "listen": [
@@ -216,6 +217,7 @@ func TestH2ToH1ChunkedResponse(t *testing.T) {
     "http": {
       "http_port": 9080,
       "https_port": 9443,
+	  "grace_period": 1,
       "servers": {
         "srv0": {
           "listen": [