package transformer_test

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/clock/fakeclock"
	mfakes "code.cloudfoundry.org/diego-logging-client/testhelpers"
	"code.cloudfoundry.org/executor"
	"code.cloudfoundry.org/executor/depot/log_streamer"
	"code.cloudfoundry.org/executor/depot/transformer"
	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/garden/gardenfakes"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/workpool"
	"github.com/gogo/protobuf/proto"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var _ = Describe("Transformer", func() {
	Describe("StepsRunner", func() {
		var (
			logger                      lager.Logger
			optimusPrime                transformer.Transformer
			container                   executor.Container
			logStreamer                 log_streamer.LogStreamer
			gardenContainer             *gardenfakes.FakeContainer
			clock                       *fakeclock.FakeClock
			fakeMetronClient            *mfakes.FakeIngressClient
			healthyMonitoringInterval   time.Duration
			unhealthyMonitoringInterval time.Duration
			gracefulShutdownInterval    time.Duration
			healthCheckWorkPool         *workpool.WorkPool
			ldsPort                     uint16
			cfg                         transformer.Config
			options                     []transformer.Option
		)

		BeforeEach(func() {
			gardenContainer = &gardenfakes.FakeContainer{}
			fakeMetronClient = &mfakes.FakeIngressClient{}

			logger = lagertest.NewTestLogger("test-container-store")
			logStreamer = log_streamer.New("test", "test", 1, fakeMetronClient)

			healthyMonitoringInterval = 1 * time.Second
			unhealthyMonitoringInterval = 1 * time.Millisecond
			gracefulShutdownInterval = 10 * time.Second

			var err error
			healthCheckWorkPool, err = workpool.NewWorkPool(10)
			Expect(err).NotTo(HaveOccurred())

			clock = fakeclock.NewFakeClock(time.Now())

			ldsPort = 65535
			cfg = transformer.Config{
				LDSPort: ldsPort,
				BindMounts: []garden.BindMount{
					{
						SrcPath: "/some/source",
						DstPath: "/some/destintation",
						Mode:    garden.BindMountModeRO,
						Origin:  garden.BindMountOriginHost,
					},
				},
			}

			options = []transformer.Option{
				transformer.WithPostSetupHook(
					"jim",
					[]string{"/post-setup/path", "-x", "argument"},
				),
				transformer.WithSidecarRootfs("preloaded:cflinuxfs2"),
			}

			container = executor.Container{
				RunInfo: executor.RunInfo{
					Setup: &models.Action{
						RunAction: &models.RunAction{
							Path: "/setup/path",
						},
					},
					Action: &models.Action{
						RunAction: &models.RunAction{
							Path: "/action/path",
						},
					},
					Monitor: &models.Action{
						RunAction: &models.RunAction{
							Path: "/monitor/path",
						},
					},
				},
			}
		})

		JustBeforeEach(func() {
			optimusPrime = transformer.NewTransformer(
				clock,
				nil, nil, nil, nil, nil,
				os.TempDir(),
				healthyMonitoringInterval,
				unhealthyMonitoringInterval,
				gracefulShutdownInterval,
				healthCheckWorkPool,
				options...,
			)
		})

		Context("when there is no run action", func() {
			BeforeEach(func() {
				container.Action = nil
			})

			It("returns an error", func() {
				_, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).To(HaveOccurred())
			})
		})

		It("returns a step encapsulating setup, post-setup, action, and monitor", func() {
			setupReceived := make(chan struct{})
			postSetupReceived := make(chan struct{})
			gardenContainer.RunStub = func(processSpec garden.ProcessSpec, processIO garden.ProcessIO) (garden.Process, error) {
				if processSpec.Path == "/setup/path" {
					setupReceived <- struct{}{}
				} else if processSpec.Path == "/post-setup/path" {
					postSetupReceived <- struct{}{}
				}
				return &gardenfakes.FakeProcess{}, nil
			}

			runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
			Expect(err).NotTo(HaveOccurred())

			process := ifrit.Background(runner)

			Eventually(gardenContainer.RunCallCount).Should(Equal(1))
			processSpec, _ := gardenContainer.RunArgsForCall(0)
			Expect(processSpec.Path).To(Equal("/setup/path"))
			Consistently(gardenContainer.RunCallCount).Should(Equal(1))

			<-setupReceived

			Eventually(gardenContainer.RunCallCount).Should(Equal(2))
			processSpec, _ = gardenContainer.RunArgsForCall(1)
			Expect(processSpec.Path).To(Equal("/post-setup/path"))
			Expect(processSpec.Args).To(Equal([]string{"-x", "argument"}))
			Expect(processSpec.User).To(Equal("jim"))
			Consistently(gardenContainer.RunCallCount).Should(Equal(2))

			<-postSetupReceived

			Eventually(gardenContainer.RunCallCount).Should(Equal(3))
			processSpec, _ = gardenContainer.RunArgsForCall(2)
			Expect(processSpec.Path).To(Equal("/action/path"))
			Consistently(gardenContainer.RunCallCount).Should(Equal(3))

			clock.Increment(1 * time.Second)
			Eventually(gardenContainer.RunCallCount).Should(Equal(4))
			processSpec, processIO := gardenContainer.RunArgsForCall(3)
			Expect(processSpec.Path).To(Equal("/monitor/path"))
			Expect(container.Monitor.RunAction.GetSuppressLogOutput()).Should(BeFalse())
			Expect(processIO.Stdout).ShouldNot(Equal(ioutil.Discard))

			process.Signal(os.Interrupt)
			clock.Increment(1 * time.Second)
			Eventually(process.Wait()).Should(Receive(nil))
		})

		It("does not become ready until the healthcheck passes", func() {
			monitorProcess := &gardenfakes.FakeProcess{}
			monitorProcess.WaitStub = func() (int, error) {
				if monitorProcess.WaitCallCount() == 1 {
					return 1, errors.New("boom")
				}
				return 0, nil
			}
			gardenContainer.RunStub = func(processSpec garden.ProcessSpec, processIO garden.ProcessIO) (garden.Process, error) {
				if processSpec.Path == "/monitor/path" {
					return monitorProcess, nil
				}
				return &gardenfakes.FakeProcess{}, nil
			}
			runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
			Expect(err).NotTo(HaveOccurred())
			process := ifrit.Background(runner)
			Consistently(process.Ready()).ShouldNot(BeClosed())

			clock.Increment(1 * time.Second)
			Consistently(process.Ready()).ShouldNot(BeClosed())

			clock.Increment(1 * time.Second)
			Eventually(process.Ready()).Should(BeClosed())
		})

		makeProcess := func(waitCh chan int) *gardenfakes.FakeProcess {
			process := &gardenfakes.FakeProcess{}
			process.WaitStub = func() (int, error) {
				return <-waitCh, nil
			}
			return process
		}

		Describe("container proxy", func() {
			var (
				container    executor.Container
				processLock  sync.Mutex
				process      ifrit.Process
				envoyProcess *gardenfakes.FakeProcess
				actionCh     chan int
				envoyCh      chan int
			)

			BeforeEach(func() {
				options = append(options, transformer.WithContainerProxy(time.Second))

				processLock.Lock()
				defer processLock.Unlock()

				actionCh = make(chan int, 1)
				actionProcess := makeProcess(actionCh)

				envoyCh = make(chan int)
				envoyProcess = makeProcess(envoyCh)

				gardenContainer.RunStub = func(spec garden.ProcessSpec, io garden.ProcessIO) (process garden.Process, err error) {
					defer GinkgoRecover()

					processLock.Lock()
					defer processLock.Unlock()

					switch spec.Path {
					case "/action/path":
						return actionProcess, nil
					case "sh":
						return envoyProcess, nil
					}

					err = errors.New("")
					Fail("unexpected executable path: " + spec.Path)
					return nil, err
				}

				container = executor.Container{
					ExternalIP: "10.0.0.1",
					InternalIP: "11.0.0.1",
					RunInfo: executor.RunInfo{
						Action: &models.Action{
							RunAction: &models.RunAction{
								Path: "/action/path",
							},
						},
						Monitor: &models.Action{
							RunAction: &models.RunAction{
								Path: "/monitor/path",
							},
						},
						CheckDefinition: &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             5432,
										RequestTimeoutMs: 100,
										Path:             "/some/path",
									},
								},
							},
						},
						Ports: []executor.PortMapping{
							executor.PortMapping{
								HostPort:      61001,
								ContainerPort: 8080,
							},
							executor.PortMapping{
								HostPort:      61002,
								ContainerPort: 61001,
							},
						},
						EnableContainerProxy: true,
					},
				}
			})

			JustBeforeEach(func() {
				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())

				process = ifrit.Background(runner)
			})

			AfterEach(func() {
				close(actionCh)
				close(envoyCh)
				ginkgomon.Interrupt(process)
			})

			It("runs the container proxy in a sidecar container", func() {
				Eventually(gardenContainer.RunCallCount).Should(Equal(2))
				specs := []garden.ProcessSpec{}
				for i := 0; i < gardenContainer.RunCallCount(); i++ {
					spec, _ := gardenContainer.RunArgsForCall(i)
					specs = append(specs, spec)
				}

				Expect(specs).To(ContainElement(garden.ProcessSpec{
					ID:   fmt.Sprintf("%s-envoy", gardenContainer.Handle()),
					Path: "sh",
					Args: []string{
						"-c",
						"trap 'kill -9 0' TERM; /etc/cf-assets/envoy/envoy -c /etc/cf-assets/envoy_config/envoy.yaml --service-cluster proxy-cluster --service-node proxy-node --drain-time-s 1 --log-level critical& pid=$!; wait $pid",
					},

					Env: []string{
						"CF_INSTANCE_IP=10.0.0.1",
						"CF_INSTANCE_INTERNAL_IP=11.0.0.1",
						"CF_INSTANCE_PORT=61001",
						"CF_INSTANCE_ADDR=10.0.0.1:61001",
						"CF_INSTANCE_PORTS=[{\"external\":61001,\"internal\":8080},{\"external\":61002,\"internal\":61001}]",
					},
					Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
					BindMounts: []garden.BindMount{
						{
							SrcPath: "/some/source",
							DstPath: "/some/destintation",
							Mode:    garden.BindMountModeRO,
							Origin:  garden.BindMountOriginHost,
						},
					},
				}))
			})

			Context("when the process is signalled", func() {
				JustBeforeEach(func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					process.Signal(os.Interrupt)
				})

				It("does not signal the proxy process", func() {
					Consistently(envoyProcess.SignalCallCount).Should(BeZero())
				})
			})

			Context("when the envoy process is signalled", func() {
				JustBeforeEach(func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					envoyCh <- 134
				})

				It("logs the exit status", func() {
					Eventually(fakeMetronClient.SendAppLogCallCount).Should(Equal(2))
					_, msg0, _, _ := fakeMetronClient.SendAppLogArgsForCall(0)
					_, msg1, _, _ := fakeMetronClient.SendAppLogArgsForCall(1)
					Expect([]string{msg0, msg1}).To(ContainElement("Exit status 134"))
				})

				It("process should fail with a descriptive error", func() {
					actionCh <- 0
					Eventually(process.Wait()).Should(Receive(MatchError("PROXY: Exited with status 134")))
				})
			})

			Context("when the container is privileged", func() {
				BeforeEach(func() {
					container.Privileged = true
				})

				It("runs the container proxy in a sidecar container", func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					specs := []garden.ProcessSpec{}
					for i := 0; i < gardenContainer.RunCallCount(); i++ {
						spec, _ := gardenContainer.RunArgsForCall(i)
						specs = append(specs, spec)
					}

					Expect(specs).To(ContainElement(garden.ProcessSpec{
						ID:   fmt.Sprintf("%s-envoy", gardenContainer.Handle()),
						Path: "sh",
						Args: []string{
							"-c",
							"trap 'kill -9 0' TERM; /etc/cf-assets/envoy/envoy -c /etc/cf-assets/envoy_config/envoy.yaml --service-cluster proxy-cluster --service-node proxy-node --drain-time-s 1 --log-level critical& pid=$!; wait $pid",
						},

						Env: []string{
							"CF_INSTANCE_IP=10.0.0.1",
							"CF_INSTANCE_INTERNAL_IP=11.0.0.1",
							"CF_INSTANCE_PORT=61001",
							"CF_INSTANCE_ADDR=10.0.0.1:61001",
							"CF_INSTANCE_PORTS=[{\"external\":61001,\"internal\":8080},{\"external\":61002,\"internal\":61001}]",
						},
						Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
						BindMounts: []garden.BindMount{
							{
								SrcPath: "/some/source",
								DstPath: "/some/destintation",
								Mode:    garden.BindMountModeRO,
								Origin:  garden.BindMountOriginHost,
							},
						},
					}))
				})
			})

			It("runs the listener discovery service (lds) in a sidecar", func() {
				Eventually(gardenContainer.RunCallCount).Should(Equal(2))
				specs := []garden.ProcessSpec{}
				for i := 0; i < gardenContainer.RunCallCount(); i++ {
					spec, _ := gardenContainer.RunArgsForCall(i)
					specs = append(specs, spec)
				}

				Expect(specs).NotTo(ContainElement(garden.ProcessSpec{
					ID:   fmt.Sprintf("%s-lds", gardenContainer.Handle()),
					Path: "/etc/cf-assets/envoy/lds",
					Args: []string{
						fmt.Sprintf("-port=%d", ldsPort),
						"-listener-config=/etc/cf-assets/envoy_config/listeners.json",
					},
					Env: []string{},
					Limits: garden.ResourceLimits{
						Nofile: proto.Uint64(1024),
					},
					Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
					BindMounts: []garden.BindMount{
						{
							SrcPath: "/some/source",
							DstPath: "/some/destintation",
							Mode:    garden.BindMountModeRO,
							Origin:  garden.BindMountOriginHost,
						},
					},
				}))
			})

			Context("when the container is privileged", func() {
				BeforeEach(func() {
					container.Privileged = true
				})

				It("runs the listener discovery service (lds) in the same container", func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					specs := []garden.ProcessSpec{}
					for i := 0; i < gardenContainer.RunCallCount(); i++ {
						spec, _ := gardenContainer.RunArgsForCall(i)
						specs = append(specs, spec)
					}

					Expect(specs).NotTo(ContainElement(garden.ProcessSpec{
						ID:   "",
						Path: "/etc/cf-assets/envoy/lds",
						Args: []string{
							fmt.Sprintf("-port=%d", ldsPort),
							"-listener-config=/etc/cf-assets/envoy_config/listeners.json",
						},
						Env: []string{},
						Limits: garden.ResourceLimits{
							Nofile: proto.Uint64(1024),
						},
					}))
				})
			})

			Context("when the container proxy is disabled on the container", func() {
				BeforeEach(func() {
					container.EnableContainerProxy = false
				})

				It("does not run the container proxy", func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(1))
					paths := []string{}
					for i := 0; i < gardenContainer.RunCallCount(); i++ {
						spec, _ := gardenContainer.RunArgsForCall(i)
						paths = append(paths, spec.Path)
					}

					Expect(paths).NotTo(ContainElement("sh"))
				})
			})
		})

		Describe("declarative healthchecks", func() {
			var (
				process                       ifrit.Process
				readinessProcess              *gardenfakes.FakeProcess
				readinessCh                   chan int
				livenessProcess               *gardenfakes.FakeProcess
				livenessCh                    chan int
				actionProcess                 *gardenfakes.FakeProcess
				actionCh                      chan int
				monitorProcess                *gardenfakes.FakeProcess
				monitorCh                     chan int
				readinessIO                   chan garden.ProcessIO
				livenessIO                    chan garden.ProcessIO
				processLock                   sync.Mutex
				specs                         chan garden.ProcessSpec
				declarativeHealthcheckSrcPath string = filepath.Join(string(os.PathSeparator), "dir", "healthcheck")
			)

			BeforeEach(func() {

				processLock.Lock()
				defer processLock.Unlock()

				readinessIO = make(chan garden.ProcessIO, 1)
				livenessIO = make(chan garden.ProcessIO, 1)
				specs = make(chan garden.ProcessSpec, 10)

				readinessIOCh := readinessIO
				livenessIOCh := livenessIO
				specsCh := specs

				readinessCh = make(chan int)
				readinessProcess = makeProcess(readinessCh)

				livenessCh = make(chan int, 1)
				livenessProcess = makeProcess(livenessCh)

				actionCh = make(chan int, 1)
				actionProcess = makeProcess(actionCh)

				monitorCh = make(chan int)
				monitorProcess = makeProcess(monitorCh)

				healthcheckCallCount := int64(0)
				gardenContainer.RunStub = func(spec garden.ProcessSpec, io garden.ProcessIO) (process garden.Process, err error) {
					specsCh <- spec

					defer GinkgoRecover()

					processLock.Lock()
					defer processLock.Unlock()

					switch spec.Path {
					case "/action/path":
						return actionProcess, nil
					case filepath.Join(transformer.HealthCheckDstPath, "healthcheck"):
						oldCount := atomic.AddInt64(&healthcheckCallCount, 1)
						switch oldCount {
						case 1:
							readinessIOCh <- io
							return readinessProcess, nil
						case 2:
							livenessIOCh <- io
							return livenessProcess, nil
						}
					case "/monitor/path":
						return monitorProcess, nil
					}

					err = errors.New("")
					Fail("unexpected executable path: " + spec.Path)
					return
				}
				container = executor.Container{
					RunInfo: executor.RunInfo{
						StartTimeoutMs: 1000,
						Action: &models.Action{
							RunAction: &models.RunAction{
								Path: "/action/path",
							},
						},
						Monitor: &models.Action{
							RunAction: &models.RunAction{
								Path: "/monitor/path",
							},
						},
						CheckDefinition: &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             5432,
										RequestTimeoutMs: 100,
										Path:             "/some/path",
									},
								},
							},
						},
					},
				}
			})

			JustBeforeEach(func() {
				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())

				process = ifrit.Background(runner)
			})

			AfterEach(func() {
				close(readinessCh)
				livenessCh <- 1 
				close(actionCh)
				close(monitorCh)
				ginkgomon.Interrupt(process)
			})

			Context("when they are enabled", func() {
				BeforeEach(func() {
					options = append(options, transformer.WithDeclarativeHealthchecks())

					container.StartTimeoutMs = 1000
				})

				AfterEach(func() {
					process.Signal(os.Kill)
				})

				Context("and no check definitions exist", func() {
					BeforeEach(func() {
						container.CheckDefinition = nil
					})

					JustBeforeEach(func() {
						clock.WaitForWatcherAndIncrement(unhealthyMonitoringInterval)
					})

					It("uses the monitor action", func() {
						Eventually(gardenContainer.RunCallCount, 5*time.Second).Should(Equal(2))
						paths := []string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							paths = append(paths, spec.Path)
						}

						Expect(paths).To(ContainElement("/monitor/path"))
					})

					Context("and container proxy is enabled", func() {
						BeforeEach(func() {
							options = append(options, transformer.WithContainerProxy(time.Second))
							cfg.BindMounts = append(cfg.BindMounts, garden.BindMount{
								Origin:  garden.BindMountOriginHost,
								SrcPath: declarativeHealthcheckSrcPath,
								DstPath: transformer.HealthCheckDstPath,
							})
							cfg.ProxyTLSPorts = []uint16{61001}
						})

						It("runs healthchecks for the envoy proxy ports", func() {
							Eventually(specs).Should(Receive(Equal(garden.ProcessSpec{
								ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "envoy-readiness-healthcheck-0"),
								Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
								Args: []string{
									"-port=61001",
									"-timeout=1000ms",
									fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
									fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
								},
								Env: []string{
									"CF_INSTANCE_IP=",
									"CF_INSTANCE_INTERNAL_IP=",
									"CF_INSTANCE_PORT=",
									"CF_INSTANCE_ADDR=",
									"CF_INSTANCE_PORTS=[]",
								},
								Limits: garden.ResourceLimits{
									Nofile: proto.Uint64(1024),
								},
								OverrideContainerLimits: &garden.ProcessLimits{},
								Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
								BindMounts: []garden.BindMount{
									{
										SrcPath: "/some/source",
										DstPath: "/some/destintation",
										Mode:    garden.BindMountModeRO,
										Origin:  garden.BindMountOriginHost,
									},
									{
										Origin:  garden.BindMountOriginHost,
										SrcPath: declarativeHealthcheckSrcPath,
										DstPath: transformer.HealthCheckDstPath,
									},
								},
							})))
						})
					})
				})

				Context("and an http check definition exists", func() {
					BeforeEach(func() {
						cfg.BindMounts = append(cfg.BindMounts, garden.BindMount{
							Origin:  garden.BindMountOriginHost,
							SrcPath: declarativeHealthcheckSrcPath,
							DstPath: transformer.HealthCheckDstPath,
						})
						container.CheckDefinition = &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             5432,
										RequestTimeoutMs: 100,
										Path:             "/some/path",
									},
								},
							},
						}
					})

					Context("and container proxy is enabled", func() {
						BeforeEach(func() {
							options = append(options, transformer.WithContainerProxy(time.Second))
							cfg.ProxyTLSPorts = []uint16{61001}
						})

						It("runs healthchecks for the envoy proxy ports", func() {
							Eventually(specs).Should(Receive(Equal(garden.ProcessSpec{
								ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "envoy-readiness-healthcheck-0"),
								Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
								Args: []string{
									"-port=61001",
									"-timeout=1000ms",
									fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
									fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
								},
								Env: []string{
									"CF_INSTANCE_IP=",
									"CF_INSTANCE_INTERNAL_IP=",
									"CF_INSTANCE_PORT=",
									"CF_INSTANCE_ADDR=",
									"CF_INSTANCE_PORTS=[]",
								},
								Limits: garden.ResourceLimits{
									Nofile: proto.Uint64(1024),
								},
								OverrideContainerLimits: &garden.ProcessLimits{},
								Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
								BindMounts: []garden.BindMount{
									{
										SrcPath: "/some/source",
										DstPath: "/some/destintation",
										Mode:    garden.BindMountModeRO,
										Origin:  garden.BindMountOriginHost,
									},
									{
										Origin:  garden.BindMountOriginHost,
										SrcPath: declarativeHealthcheckSrcPath,
										DstPath: transformer.HealthCheckDstPath,
									},
								},
							})))
						})
					})

					It("runs the readiness healthcheck in a sidecar container", func() {
						Eventually(specs).Should(Receive(Equal(garden.ProcessSpec{
							ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-0"),
							Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
							Args: []string{
								"-port=5432",
								"-timeout=100ms",
								"-uri=/some/path",
								fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
								fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
							},
							Env: []string{
								"CF_INSTANCE_IP=",
								"CF_INSTANCE_INTERNAL_IP=",
								"CF_INSTANCE_PORT=",
								"CF_INSTANCE_ADDR=",
								"CF_INSTANCE_PORTS=[]",
							},
							Limits: garden.ResourceLimits{
								Nofile: proto.Uint64(1024),
							},
							OverrideContainerLimits: &garden.ProcessLimits{},
							Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
							BindMounts: []garden.BindMount{
								{
									SrcPath: "/some/source",
									DstPath: "/some/destintation",
									Mode:    garden.BindMountModeRO,
									Origin:  garden.BindMountOriginHost,
								},
								{
									Origin:  garden.BindMountOriginHost,
									SrcPath: declarativeHealthcheckSrcPath,
									DstPath: transformer.HealthCheckDstPath,
								},
							},
						})))
					})

					Context("when the container is privileged", func() {
						BeforeEach(func() {
							container.Privileged = true
						})

						It("runs the healthcheck in a sidecar container", func() {
							Eventually(specs).Should(Receive(Equal(garden.ProcessSpec{
								ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-0"),
								Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
								Args: []string{
									"-port=5432",
									"-timeout=100ms",
									"-uri=/some/path",
									fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
									fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
								},
								Env: []string{
									"CF_INSTANCE_IP=",
									"CF_INSTANCE_INTERNAL_IP=",
									"CF_INSTANCE_PORT=",
									"CF_INSTANCE_ADDR=",
									"CF_INSTANCE_PORTS=[]",
								},
								Limits: garden.ResourceLimits{
									Nofile: proto.Uint64(1024),
								},
								OverrideContainerLimits: &garden.ProcessLimits{},
								Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
								BindMounts: []garden.BindMount{
									{
										SrcPath: "/some/source",
										DstPath: "/some/destintation",
										Mode:    garden.BindMountModeRO,
										Origin:  garden.BindMountOriginHost,
									},
									{
										Origin:  garden.BindMountOriginHost,
										SrcPath: declarativeHealthcheckSrcPath,
										DstPath: transformer.HealthCheckDstPath,
									},
								},
							})))
						})
					})

					Context("and the starttimeout is set to 0", func() {
						BeforeEach(func() {
							container.StartTimeoutMs = 0
						})

						It("runs the healthcheck with readiness timeout set to 0", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(2))
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=100ms",
								"-uri=/some/path",
								"-readiness-interval=1ms",
								"-readiness-timeout=0s",
							}))
						})
					})

					Context("and optional fields are missing", func() {
						BeforeEach(func() {
							container.CheckDefinition = &models.CheckDefinition{
								Checks: []*models.Check{
									&models.Check{
										HttpCheck: &models.HTTPCheck{
											Port: 5432,
										},
									},
								},
							}
						})

						It("uses sane defaults", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(2))
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=1000ms",
								"-uri=/",
								"-readiness-interval=1ms",
								"-readiness-timeout=1s",
							}))
						})
					})

					It("uses the check definition", func() {
						Eventually(gardenContainer.RunCallCount).Should(Equal(2))
						paths := []string{}
						args := [][]string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							paths = append(paths, spec.Path)
							args = append(args, spec.Args)
						}

						Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
						Expect(args).To(ContainElement([]string{
							"-port=5432",
							"-timeout=100ms",
							"-uri=/some/path",
							"-readiness-interval=1ms", 
							"-readiness-timeout=1s",
						}))
					})

					Context("when the readiness check times out", func() {
						JustBeforeEach(func() {
							By("waiting for the action and readiness check processes to start")
							var io garden.ProcessIO
							Eventually(readinessIO).Should(Receive(&io))
							_, err := io.Stdout.Write([]byte("readiness check failed"))
							Expect(err).NotTo(HaveOccurred())

							By("timing out the readiness check")
							Eventually(gardenContainer.RunCallCount).Should(Equal(2))

							Consistently(readinessProcess.SignalCallCount).Should(Equal(0))
							readinessCh <- 1
							Eventually(actionProcess.SignalCallCount).Should(Equal(1))
							actionCh <- 2
						})

						It("suppress the readiness check log", func() {
							Eventually(process.Wait()).Should(Receive(HaveOccurred()))
							Consistently(fakeMetronClient.SendAppLogCallCount).Should(Equal(2))
							_, msg0, _, _ := fakeMetronClient.SendAppLogArgsForCall(0)
							_, msg1, _, _ := fakeMetronClient.SendAppLogArgsForCall(1)
							Expect([]string{msg0, msg1}).To(ConsistOf("Starting health monitoring of container", "Exit status 2"))
						})

						It("logs the readiness check output on stderr", func() {
							Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(Equal(2))
							logLines := map[string]string{}
							_, msg, source, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
							logLines[source] = msg
							_, msg, source, _ = fakeMetronClient.SendAppErrorLogArgsForCall(1)
							logLines[source] = msg
							Expect(logLines).To(Equal(map[string]string{
								"HEALTH": "readiness check failed",
								"test":   "Timed out after 1s: health check never passed.",
							}))
						})

						It("returns the readiness check output in the error", func() {
							Consistently(process.Ready()).ShouldNot(BeClosed())
						})

						It("returns the readiness check output in the error", func() {
							Eventually(process.Wait()).Should(Receive(MatchError(ContainSubstring("Instance never healthy after 1s: readiness check failed"))))
						})
					})

					Context("when the readiness check passes", func() {
						JustBeforeEach(func() {
							readinessCh <- 0
						})

						It("starts the liveness check", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(3))
							ids := []string{}
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								ids = append(ids, spec.ID)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "liveness-healthcheck-0")))
							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=100ms",
								"-uri=/some/path",
								"-liveness-interval=1s",
							}))
						})

						Context("when the liveness check exits", func() {
							JustBeforeEach(func() {
								Eventually(gardenContainer.RunCallCount).Should(Equal(3))

								By("waiting the action and liveness check processes to start")
								var io garden.ProcessIO
								Eventually(livenessIO).Should(Receive(&io))
								_, err := io.Stdout.Write([]byte("liveness check failed"))
								Expect(err).NotTo(HaveOccurred())

								By("exiting the liveness check")
								livenessCh <- 1
								Eventually(actionProcess.SignalCallCount).Should(Equal(1))
								actionCh <- 2
							})

							It("logs the liveness check output on stderr", func() {
								Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(Equal(1))
								logLines := map[string]string{}
								_, msg, source, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
								logLines[source] = msg
								Expect(logLines).To(Equal(map[string]string{
									"HEALTH": "liveness check failed",
								}))
							})

							It("returns the liveness check output in the error", func() {
								Eventually(process.Wait()).Should(Receive(MatchError(ContainSubstring("Instance became unhealthy: liveness check failed"))))
							})
						})
					})
				})

				Context("and a tcp check definition exists", func() {
					BeforeEach(func() {
						container.CheckDefinition = &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									TcpCheck: &models.TCPCheck{
										Port:             5432,
										ConnectTimeoutMs: 100,
									},
								},
							},
						}
					})

					Context("and optional fields are missing", func() {
						BeforeEach(func() {
							container.CheckDefinition = &models.CheckDefinition{
								Checks: []*models.Check{
									&models.Check{
										TcpCheck: &models.TCPCheck{
											Port: 5432,
										},
									},
								},
							}
						})

						It("uses sane defaults", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(2))
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=1000ms",
								"-readiness-interval=1ms",
								"-readiness-timeout=1s",
							}))
						})
					})

					It("uses the readiness check definition", func() {
						Eventually(gardenContainer.RunCallCount).Should(Equal(2))
						ids := []string{}
						paths := []string{}
						args := [][]string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							ids = append(ids, spec.ID)
							paths = append(paths, spec.Path)
							args = append(args, spec.Args)
						}

						Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-0")))
						Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
						Expect(args).To(ContainElement([]string{
							"-port=5432",
							"-timeout=100ms",
							"-readiness-interval=1ms",
							"-readiness-timeout=1s",
						}))
					})

					Context("when the readiness check passes", func() {
						JustBeforeEach(func() {
							readinessCh <- 0
						})

						It("uses the liveness check definition", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(3))
							ids := []string{}
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								ids = append(ids, spec.ID)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "liveness-healthcheck-0")))
							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=100ms",
								"-liveness-interval=1s",
							}))

						})
					})
				})

				Context("logs", func() {
					BeforeEach(func() {
						container.CheckDefinition = &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             5432,
										RequestTimeoutMs: 2000,
										Path:             "/some/path",
									},
								},
							},
						}
					})

					JustBeforeEach(func() {
						Eventually(fakeMetronClient.SendAppLogCallCount, 5).Should(BeNumerically(">=", 1))

						_, message, sourceName, _ := fakeMetronClient.SendAppLogArgsForCall(0)
						Expect(message).To(Equal("Starting health monitoring of container"))
						Expect(sourceName).To(Equal("test"))

						var io garden.ProcessIO
						Eventually(readinessIO).Should(Receive(&io))
						io.Stdout.Write([]byte("failed"))

						readinessCh <- 1
					})

					It("should default to HEALTH for log source", func() {
						Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(BeNumerically(">=", 1))
						_, _, sourceName, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
						Expect(sourceName).To(Equal("HEALTH"))
					})

					Context("when log source defined", func() {
						BeforeEach(func() {
							container.CheckDefinition.LogSource = "healthcheck"
						})

						It("logs healthcheck errors with log source from check defintion", func() {
							Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(BeNumerically(">=", 1))
							_, _, sourceName, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
							Expect(sourceName).To(Equal("healthcheck"))
						})
					})
				})

				Context("and multiple check definitions exists", func() {
					var (
						otherReadinessProcess *gardenfakes.FakeProcess
						otherReadinessCh      chan int
						otherLivenessProcess  *gardenfakes.FakeProcess
						otherLivenessCh       chan int
					)

					BeforeEach(func() {

						processLock.Lock()
						defer processLock.Unlock()

						otherReadinessCh = make(chan int)
						otherReadinessProcess = makeProcess(otherReadinessCh)

						otherLivenessCh = make(chan int)
						otherLivenessProcess = makeProcess(otherLivenessCh)

						healthcheckCallCount := int64(0)
						gardenContainer.RunStub = func(spec garden.ProcessSpec, io garden.ProcessIO) (process garden.Process, err error) {
							defer GinkgoRecover()

							processLock.Lock()
							defer processLock.Unlock()

							switch spec.Path {
							case "/action/path":
								return actionProcess, nil
							case filepath.Join(transformer.HealthCheckDstPath, "healthcheck"):
								oldCount := atomic.AddInt64(&healthcheckCallCount, 1)
								switch oldCount {
								case 1:
									return readinessProcess, nil
								case 2:
									return otherReadinessProcess, nil
								case 3:
									return livenessProcess, nil
								case 4:
									return otherLivenessProcess, nil
								}
								return livenessProcess, nil
							case "/monitor/path":
								return monitorProcess, nil
							}

							err = errors.New("")
							Fail("unexpected executable path: " + spec.Path)
							return
						}

						container.CheckDefinition = &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									TcpCheck: &models.TCPCheck{
										Port:             2222,
										ConnectTimeoutMs: 100,
									},
								},
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             8080,
										RequestTimeoutMs: 100,
									},
								},
							},
						}
					})

					AfterEach(func() {
						close(otherReadinessCh)
						close(otherLivenessCh)
					})

					It("uses the check definition instead of the monitor action", func() {
						Eventually(gardenContainer.RunCallCount).Should(Equal(3))
						ids := []string{}
						paths := []string{}
						args := [][]string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							ids = append(ids, spec.ID)
							paths = append(paths, spec.Path)
							args = append(args, spec.Args)
						}

						Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-0")))
						Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-1")))

						Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
						Expect(args).To(ContainElement([]string{
							"-port=2222",
							"-timeout=100ms",
							"-readiness-interval=1ms",
							"-readiness-timeout=1s",
						}))
						Expect(args).To(ContainElement([]string{
							"-port=8080",
							"-timeout=100ms",
							"-uri=/",
							"-readiness-interval=1ms",
							"-readiness-timeout=1s",
						}))
					})

					Context("when one of the readiness checks finish", func() {
						JustBeforeEach(func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(3))
							readinessCh <- 0
						})

						It("waits for both healthchecks to pass", func() {
							Consistently(gardenContainer.RunCallCount).Should(Equal(3))
						})

						Context("and the other readiness check finish", func() {
							JustBeforeEach(func() {
								otherReadinessCh <- 0
							})

							It("starts the liveness checks", func() {
								Eventually(gardenContainer.RunCallCount).Should(Equal(5))
								ids := []string{}
								paths := []string{}
								args := [][]string{}
								for i := 0; i < gardenContainer.RunCallCount(); i++ {
									spec, _ := gardenContainer.RunArgsForCall(i)
									ids = append(ids, spec.ID)
									paths = append(paths, spec.Path)
									args = append(args, spec.Args)
								}

								Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "liveness-healthcheck-0")))
								Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "liveness-healthcheck-1")))

								Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
								Expect(args).To(ContainElement([]string{
									"-port=2222",
									"-timeout=100ms",
									"-liveness-interval=1s",
								}))
								Expect(args).To(ContainElement([]string{
									"-port=8080",
									"-timeout=100ms",
									"-uri=/",
									"-liveness-interval=1s",
								}))
							})

							Context("when either liveness check exit", func() {
								JustBeforeEach(func() {
									Eventually(gardenContainer.RunCallCount).Should(Equal(5))
									livenessCh <- 1
								})

								It("signals the process and exit", func() {
									Eventually(otherLivenessProcess.SignalCallCount).ShouldNot(BeZero())
									otherLivenessCh <- 1

									Eventually(actionProcess.SignalCallCount).ShouldNot(BeZero())
									actionCh <- 0

									Eventually(process.Wait()).Should(Receive(HaveOccurred()))
								})
							})
						})
					})
				})
			})

			Context("when they are disabled", func() {
				It("ignores the check definition and use the MonitorAction", func() {
					clock.WaitForWatcherAndIncrement(unhealthyMonitoringInterval)
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					paths := []string{}
					for i := 0; i < gardenContainer.RunCallCount(); i++ {
						spec, _ := gardenContainer.RunArgsForCall(i)
						paths = append(paths, spec.Path)
					}

					Expect(paths).To(ContainElement("/monitor/path"))
				})

				Context("and container proxy is enabled", func() {
					BeforeEach(func() {
						options = append(options, transformer.WithContainerProxy(time.Second))
						cfg.BindMounts = append(cfg.BindMounts, garden.BindMount{
							Origin:  garden.BindMountOriginHost,
							SrcPath: declarativeHealthcheckSrcPath,
							DstPath: transformer.HealthCheckDstPath,
						})
						cfg.ProxyTLSPorts = []uint16{61001}
					})

					It("does not run healthchecks for the envoy proxy ports", func() {
						Consistently(specs).ShouldNot(Receive(Equal(garden.ProcessSpec{
							ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "envoy-readiness-healthcheck-0"),
							Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
							Args: []string{
								"-port=61001",
								"-timeout=1000ms",
								fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
								fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
							},
							Env: []string{},
							Limits: garden.ResourceLimits{
								Nofile: proto.Uint64(1024),
							},
							OverrideContainerLimits: &garden.ProcessLimits{},
							Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
							BindMounts: []garden.BindMount{
								{
									SrcPath: "/some/source",
									DstPath: "/some/destintation",
									Mode:    garden.BindMountModeRO,
									Origin:  garden.BindMountOriginHost,
								},
								{
									Origin:  garden.BindMountOriginHost,
									SrcPath: declarativeHealthcheckSrcPath,
									DstPath: transformer.HealthCheckDstPath,
								},
							},
						})))
					})
				})

				Context("and there is no monitor action", func() {
					BeforeEach(func() {
						container.Monitor = nil
					})

					It("does not run any healthchecks", func() {
						Eventually(gardenContainer.RunCallCount).Should(Equal(1))
						Consistently(gardenContainer.RunCallCount).Should(Equal(1))

						paths := []string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							paths = append(paths, spec.Path)
						}

						Expect(paths).To(ContainElement("/action/path"))
					})
				})
			})
		})

		Context("when there is no setup", func() {
			BeforeEach(func() {
				container.Setup = nil
			})

			It("returns a codependent step for the action/monitor", func() {
				gardenContainer.RunReturns(&gardenfakes.FakeProcess{}, nil)

				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())

				process := ifrit.Background(runner)

				Eventually(gardenContainer.RunCallCount).Should(Equal(1))
				processSpec, _ := gardenContainer.RunArgsForCall(0)
				Expect(processSpec.Path).To(Equal("/action/path"))
				Consistently(gardenContainer.RunCallCount).Should(Equal(1))

				clock.Increment(1 * time.Second)
				Eventually(gardenContainer.RunCallCount).Should(Equal(2))
				processSpec, _ = gardenContainer.RunArgsForCall(1)
				Expect(processSpec.Path).To(Equal("/monitor/path"))
				Eventually(process.Ready()).Should(BeClosed())

				process.Signal(os.Interrupt)
				clock.Increment(1 * time.Second)
				Eventually(process.Wait()).Should(Receive(nil))
			})
		})

		Context("when there is no monitor", func() {
			BeforeEach(func() {
				container.Monitor = nil
				container.Setup = nil
			})

			It("does not run the monitor step and immediately says the healthcheck passed", func() {
				blockCh := make(chan struct{})
				defer close(blockCh)

				fakeGardenProcess := &gardenfakes.FakeProcess{}
				fakeGardenProcess.WaitStub = func() (int, error) {
					<-blockCh
					return 0, nil
				}
				gardenContainer.RunReturns(fakeGardenProcess, nil)

				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())

				process := ifrit.Background(runner)
				Eventually(process.Ready()).Should(BeClosed())

				Eventually(gardenContainer.RunCallCount).Should(Equal(1))
				processSpec, _ := gardenContainer.RunArgsForCall(0)
				Expect(processSpec.Path).To(Equal("/action/path"))
				Consistently(gardenContainer.RunCallCount).Should(Equal(1))
			})
		})

		Context("MonitorAction", func() {
			var (
				process ifrit.Process
			)

			JustBeforeEach(func() {
				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())
				process = ifrit.Background(runner)
			})

			AfterEach(func() {
				ginkgomon.Interrupt(process)
			})

			BeforeEach(func() {
				container.Setup = nil
				container.Monitor = &models.Action{
					ParallelAction: models.Parallel(
						&models.RunAction{
							Path:              "/monitor/path",
							SuppressLogOutput: true,
						},
						&models.RunAction{
							Path:              "/monitor/path",
							SuppressLogOutput: true,
						},
					),
				}
			})

			Context("SuppressLogOutput", func() {
				var (
					monitorCh, actionCh chan int
				)

				BeforeEach(func() {
					monitorCh = make(chan int, 2)
					actionCh = make(chan int, 1)

					gardenContainer.RunStub = func(processSpec garden.ProcessSpec, processIO garden.ProcessIO) (garden.Process, error) {
						switch processSpec.Path {
						case "/monitor/path":
							return makeProcess(monitorCh), nil
						case "/action/path":
							return makeProcess(actionCh), nil
						default:
							return &gardenfakes.FakeProcess{}, nil
						}
					}
				})

				AfterEach(func() {
					close(monitorCh)
					close(actionCh)
				})

				JustBeforeEach(func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(1))
					clock.Increment(1 * time.Second)
					Eventually(gardenContainer.RunCallCount).Should(Equal(3))
				})

				It("is ignored", func() {
					processSpec, processIO := gardenContainer.RunArgsForCall(1)
					Expect(processSpec.Path).To(Equal("/monitor/path"))
					Expect(container.Monitor.RunAction.GetSuppressLogOutput()).Should(BeFalse())
					Expect(processIO.Stdout).ShouldNot(Equal(ioutil.Discard))
					monitorCh <- 0
					monitorCh <- 0
					Eventually(process.Ready()).Should(BeClosed())
				})
			})

			Context("logs", func() {
				var (
					exitStatusCh    chan int
					monitorProcess1 *gardenfakes.FakeProcess
					monitorProcess2 *gardenfakes.FakeProcess
				)

				BeforeEach(func() {
					monitorProcess1 = &gardenfakes.FakeProcess{}
					monitorProcess2 = &gardenfakes.FakeProcess{}
					actionProcess := &gardenfakes.FakeProcess{}
					exitStatusCh = make(chan int)
					actionProcess.WaitStub = func() (int, error) {
						return <-exitStatusCh, nil
					}

					monitorProcessChan1 := make(chan *garden.ProcessIO, 4)
					monitorProcessChan2 := make(chan *garden.ProcessIO, 4)

					monitorProcess1.WaitStub = func() (int, error) {
						procIO := <-monitorProcessChan1

						if monitorProcess1.WaitCallCount() == 2 {
							procIO.Stdout.Write([]byte("healthcheck failed"))
							return 1, nil
						}
						return 0, nil
					}

					monitorProcess2.WaitStub = func() (int, error) {
						procIO := <-monitorProcessChan2

						if monitorProcess2.WaitCallCount() == 2 {
							procIO.Stdout.Write([]byte("healthcheck failed"))
							return 1, nil
						}
						return 0, nil
					}

					monitorProcessRun := uint32(0)

					gardenContainer.RunStub = func(processSpec garden.ProcessSpec, processIO garden.ProcessIO) (garden.Process, error) {
						if processSpec.Path == "/monitor/path" {
							if atomic.AddUint32(&monitorProcessRun, 1)%2 == 0 {
								monitorProcessChan1 <- &processIO
								return monitorProcess1, nil
							}
							monitorProcessChan2 <- &processIO
							return monitorProcess2, nil
						} else if processSpec.Path == "/action/path" {
							return actionProcess, nil
						}
						return &gardenfakes.FakeProcess{}, nil
					}
				})

				AfterEach(func() {
					Eventually(exitStatusCh).Should(BeSent(1))
				})

				JustBeforeEach(func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(1))

					By("starting the readiness check")
					clock.WaitForWatcherAndIncrement(1 * time.Second)
					Eventually(gardenContainer.RunCallCount).Should(Equal(3))
					Eventually(monitorProcess1.WaitCallCount).Should(Equal(1))
					Eventually(monitorProcess2.WaitCallCount).Should(Equal(1))

					By("starting the liveness check")
					clock.WaitForWatcherAndIncrement(1 * time.Second)
					Eventually(gardenContainer.RunCallCount).Should(Equal(5))
					Eventually(monitorProcess1.WaitCallCount).Should(Equal(2))
					Eventually(monitorProcess2.WaitCallCount).Should(Equal(2))
				})

				It("logs healthcheck error with the same source in a readable way", func() {
					Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(Equal(1))
					_, message, sourceName, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
					Expect(sourceName).To(Equal("test"))
					Expect(message).To(ContainSubstring("healthcheck failed; healthcheck failed"))
				})

				It("logs the container lifecycle", func() {
					Eventually(fakeMetronClient.SendAppLogCallCount).Should(Equal(3))
					_, message, _, _ := fakeMetronClient.SendAppLogArgsForCall(0)
					Expect(message).To(Equal("Starting health monitoring of container"))
					_, message, _, _ = fakeMetronClient.SendAppLogArgsForCall(1)
					Expect(message).To(Equal("Container became healthy"))
					_, message, _, _ = fakeMetronClient.SendAppLogArgsForCall(2)
					Expect(message).To(Equal("Container became unhealthy"))
				})
			})
		})
	})
})
package transformer_test

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"

	"code.cloudfoundry.org/bbs/models"
	"code.cloudfoundry.org/clock/fakeclock"
	mfakes "code.cloudfoundry.org/diego-logging-client/testhelpers"
	"code.cloudfoundry.org/executor"
	"code.cloudfoundry.org/executor/depot/log_streamer"
	"code.cloudfoundry.org/executor/depot/transformer"
	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/garden/gardenfakes"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/workpool"
	"github.com/gogo/protobuf/proto"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var _ = Describe("Transformer", func() {
	Describe("StepsRunner", func() {
		var (
			logger                      lager.Logger
			optimusPrime                transformer.Transformer
			container                   executor.Container
			logStreamer                 log_streamer.LogStreamer
			gardenContainer             *gardenfakes.FakeContainer
			clock                       *fakeclock.FakeClock
			fakeMetronClient            *mfakes.FakeIngressClient
			healthyMonitoringInterval   time.Duration
			unhealthyMonitoringInterval time.Duration
			gracefulShutdownInterval    time.Duration
			healthCheckWorkPool         *workpool.WorkPool
			ldsPort                     uint16
			cfg                         transformer.Config
			options                     []transformer.Option
		)

		BeforeEach(func() {
			gardenContainer = &gardenfakes.FakeContainer{}
			fakeMetronClient = &mfakes.FakeIngressClient{}

			logger = lagertest.NewTestLogger("test-container-store")
			logStreamer = log_streamer.New("test", "test", 1, fakeMetronClient)

			healthyMonitoringInterval = 1 * time.Second
			unhealthyMonitoringInterval = 1 * time.Millisecond
			gracefulShutdownInterval = 10 * time.Second

			var err error
			healthCheckWorkPool, err = workpool.NewWorkPool(10)
			Expect(err).NotTo(HaveOccurred())

			clock = fakeclock.NewFakeClock(time.Now())

			ldsPort = 65535
			cfg = transformer.Config{
				LDSPort: ldsPort,
				BindMounts: []garden.BindMount{
					{
						SrcPath: "/some/source",
						DstPath: "/some/destintation",
						Mode:    garden.BindMountModeRO,
						Origin:  garden.BindMountOriginHost,
					},
				},
			}

			options = []transformer.Option{
				transformer.WithPostSetupHook(
					"jim",
					[]string{"/post-setup/path", "-x", "argument"},
				),
				transformer.WithSidecarRootfs("preloaded:cflinuxfs2"),
			}

			container = executor.Container{
				RunInfo: executor.RunInfo{
					Setup: &models.Action{
						RunAction: &models.RunAction{
							Path: "/setup/path",
						},
					},
					Action: &models.Action{
						RunAction: &models.RunAction{
							Path: "/action/path",
						},
					},
					Monitor: &models.Action{
						RunAction: &models.RunAction{
							Path: "/monitor/path",
						},
					},
				},
			}
		})

		JustBeforeEach(func() {
			optimusPrime = transformer.NewTransformer(
				clock,
				nil, nil, nil, nil, nil,
				os.TempDir(),
				healthyMonitoringInterval,
				unhealthyMonitoringInterval,
				gracefulShutdownInterval,
				healthCheckWorkPool,
				options...,
			)
		})

		Context("when there is no run action", func() {
			BeforeEach(func() {
				container.Action = nil
			})

			It("returns an error", func() {
				_, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).To(HaveOccurred())
			})
		})

		It("returns a step encapsulating setup, post-setup, action, and monitor", func() {
			setupReceived := make(chan struct{})
			postSetupReceived := make(chan struct{})
			gardenContainer.RunStub = func(processSpec garden.ProcessSpec, processIO garden.ProcessIO) (garden.Process, error) {
				if processSpec.Path == "/setup/path" {
					setupReceived <- struct{}{}
				} else if processSpec.Path == "/post-setup/path" {
					postSetupReceived <- struct{}{}
				}
				return &gardenfakes.FakeProcess{}, nil
			}

			runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
			Expect(err).NotTo(HaveOccurred())

			process := ifrit.Background(runner)

			Eventually(gardenContainer.RunCallCount).Should(Equal(1))
			processSpec, _ := gardenContainer.RunArgsForCall(0)
			Expect(processSpec.Path).To(Equal("/setup/path"))
			Consistently(gardenContainer.RunCallCount).Should(Equal(1))

			<-setupReceived

			Eventually(gardenContainer.RunCallCount).Should(Equal(2))
			processSpec, _ = gardenContainer.RunArgsForCall(1)
			Expect(processSpec.Path).To(Equal("/post-setup/path"))
			Expect(processSpec.Args).To(Equal([]string{"-x", "argument"}))
			Expect(processSpec.User).To(Equal("jim"))
			Consistently(gardenContainer.RunCallCount).Should(Equal(2))

			<-postSetupReceived

			Eventually(gardenContainer.RunCallCount).Should(Equal(3))
			processSpec, _ = gardenContainer.RunArgsForCall(2)
			Expect(processSpec.Path).To(Equal("/action/path"))
			Consistently(gardenContainer.RunCallCount).Should(Equal(3))

			clock.Increment(1 * time.Second)
			Eventually(gardenContainer.RunCallCount).Should(Equal(4))
			processSpec, processIO := gardenContainer.RunArgsForCall(3)
			Expect(processSpec.Path).To(Equal("/monitor/path"))
			Expect(container.Monitor.RunAction.GetSuppressLogOutput()).Should(BeFalse())
			Expect(processIO.Stdout).ShouldNot(Equal(ioutil.Discard))

			process.Signal(os.Interrupt)
			clock.Increment(1 * time.Second)
			Eventually(process.Wait()).Should(Receive(nil))
		})

		It("does not become ready until the healthcheck passes", func() {
			monitorProcess := &gardenfakes.FakeProcess{}
			monitorProcess.WaitStub = func() (int, error) {
				if monitorProcess.WaitCallCount() == 1 {
					return 1, errors.New("boom")
				}
				return 0, nil
			}
			gardenContainer.RunStub = func(processSpec garden.ProcessSpec, processIO garden.ProcessIO) (garden.Process, error) {
				if processSpec.Path == "/monitor/path" {
					return monitorProcess, nil
				}
				return &gardenfakes.FakeProcess{}, nil
			}
			runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
			Expect(err).NotTo(HaveOccurred())
			process := ifrit.Background(runner)
			Consistently(process.Ready()).ShouldNot(BeClosed())

			clock.Increment(1 * time.Second)
			Consistently(process.Ready()).ShouldNot(BeClosed())

			clock.Increment(1 * time.Second)
			Eventually(process.Ready()).Should(BeClosed())
		})

		makeProcess := func(waitCh chan int) *gardenfakes.FakeProcess {
			process := &gardenfakes.FakeProcess{}
			process.WaitStub = func() (int, error) {
				return <-waitCh, nil
			}
			return process
		}

		Describe("container proxy", func() {
			var (
				container    executor.Container
				processLock  sync.Mutex
				process      ifrit.Process
				envoyProcess *gardenfakes.FakeProcess
				actionCh     chan int
				envoyCh      chan int
			)

			BeforeEach(func() {
				options = append(options, transformer.WithContainerProxy(time.Second))

				processLock.Lock()
				defer processLock.Unlock()

				actionCh = make(chan int, 1)
				actionProcess := makeProcess(actionCh)

				envoyCh = make(chan int)
				envoyProcess = makeProcess(envoyCh)

				gardenContainer.RunStub = func(spec garden.ProcessSpec, io garden.ProcessIO) (process garden.Process, err error) {
					defer GinkgoRecover()
					// get rid of race condition caused by write inside the BeforeEach

					processLock.Lock()
					defer processLock.Unlock()

					switch spec.Path {
					case "/action/path":
						return actionProcess, nil
					case "sh":
						return envoyProcess, nil
					}

					err = errors.New("")
					Fail("unexpected executable path: " + spec.Path)
					return nil, err
				}

				container = executor.Container{
					ExternalIP: "10.0.0.1",
					InternalIP: "11.0.0.1",
					RunInfo: executor.RunInfo{
						Action: &models.Action{
							RunAction: &models.RunAction{
								Path: "/action/path",
							},
						},
						Monitor: &models.Action{
							RunAction: &models.RunAction{
								Path: "/monitor/path",
							},
						},
						CheckDefinition: &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             5432,
										RequestTimeoutMs: 100,
										Path:             "/some/path",
									},
								},
							},
						},
						Ports: []executor.PortMapping{
							executor.PortMapping{
								HostPort:      61001,
								ContainerPort: 8080,
							},
							executor.PortMapping{
								HostPort:      61002,
								ContainerPort: 61001,
							},
						},
						EnableContainerProxy: true,
					},
				}
			})

			JustBeforeEach(func() {
				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())

				process = ifrit.Background(runner)
			})

			AfterEach(func() {
				close(actionCh)
				close(envoyCh)
				ginkgomon.Interrupt(process)
			})

			It("runs the container proxy in a sidecar container", func() {
				Eventually(gardenContainer.RunCallCount).Should(Equal(2))
				specs := []garden.ProcessSpec{}
				for i := 0; i < gardenContainer.RunCallCount(); i++ {
					spec, _ := gardenContainer.RunArgsForCall(i)
					specs = append(specs, spec)
				}

				Expect(specs).To(ContainElement(garden.ProcessSpec{
					ID:   fmt.Sprintf("%s-envoy", gardenContainer.Handle()),
					Path: "sh",
					Args: []string{
						"-c",
						"trap 'kill -9 0' TERM; /etc/cf-assets/envoy/envoy -c /etc/cf-assets/envoy_config/envoy.yaml --service-cluster proxy-cluster --service-node proxy-node --drain-time-s 1 --log-level critical& pid=$!; wait $pid",
					},

					Env: []string{
						"CF_INSTANCE_IP=10.0.0.1",
						"CF_INSTANCE_INTERNAL_IP=11.0.0.1",
						"CF_INSTANCE_PORT=61001",
						"CF_INSTANCE_ADDR=10.0.0.1:61001",
						"CF_INSTANCE_PORTS=[{\"external\":61001,\"internal\":8080},{\"external\":61002,\"internal\":61001}]",
					},
					Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
					BindMounts: []garden.BindMount{
						{
							SrcPath: "/some/source",
							DstPath: "/some/destintation",
							Mode:    garden.BindMountModeRO,
							Origin:  garden.BindMountOriginHost,
						},
					},
				}))
			})

			Context("when the process is signalled", func() {
				JustBeforeEach(func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					process.Signal(os.Interrupt)
				})

				It("does not signal the proxy process", func() {
					Consistently(envoyProcess.SignalCallCount).Should(BeZero())
				})
			})

			Context("when the envoy process is signalled", func() {
				JustBeforeEach(func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					envoyCh <- 134
				})

				It("logs the exit status", func() {
					Eventually(fakeMetronClient.SendAppLogCallCount).Should(Equal(2))
					_, msg0, _, _ := fakeMetronClient.SendAppLogArgsForCall(0)
					_, msg1, _, _ := fakeMetronClient.SendAppLogArgsForCall(1)
					Expect([]string{msg0, msg1}).To(ContainElement("Exit status 134"))
				})

				It("process should fail with a descriptive error", func() {
					actionCh <- 0
					Eventually(process.Wait()).Should(Receive(MatchError("PROXY: Exited with status 134")))
				})
			})

			Context("when the container is privileged", func() {
				BeforeEach(func() {
					container.Privileged = true
				})

				It("runs the container proxy in a sidecar container", func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					specs := []garden.ProcessSpec{}
					for i := 0; i < gardenContainer.RunCallCount(); i++ {
						spec, _ := gardenContainer.RunArgsForCall(i)
						specs = append(specs, spec)
					}

					Expect(specs).To(ContainElement(garden.ProcessSpec{
						ID:   fmt.Sprintf("%s-envoy", gardenContainer.Handle()),
						Path: "sh",
						Args: []string{
							"-c",
							"trap 'kill -9 0' TERM; /etc/cf-assets/envoy/envoy -c /etc/cf-assets/envoy_config/envoy.yaml --service-cluster proxy-cluster --service-node proxy-node --drain-time-s 1 --log-level critical& pid=$!; wait $pid",
						},

						Env: []string{
							"CF_INSTANCE_IP=10.0.0.1",
							"CF_INSTANCE_INTERNAL_IP=11.0.0.1",
							"CF_INSTANCE_PORT=61001",
							"CF_INSTANCE_ADDR=10.0.0.1:61001",
							"CF_INSTANCE_PORTS=[{\"external\":61001,\"internal\":8080},{\"external\":61002,\"internal\":61001}]",
						},
						Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
						BindMounts: []garden.BindMount{
							{
								SrcPath: "/some/source",
								DstPath: "/some/destintation",
								Mode:    garden.BindMountModeRO,
								Origin:  garden.BindMountOriginHost,
							},
						},
					}))
				})
			})

			It("runs the listener discovery service (lds) in a sidecar", func() {
				Eventually(gardenContainer.RunCallCount).Should(Equal(2))
				specs := []garden.ProcessSpec{}
				for i := 0; i < gardenContainer.RunCallCount(); i++ {
					spec, _ := gardenContainer.RunArgsForCall(i)
					specs = append(specs, spec)
				}

				Expect(specs).NotTo(ContainElement(garden.ProcessSpec{
					ID:   fmt.Sprintf("%s-lds", gardenContainer.Handle()),
					Path: "/etc/cf-assets/envoy/lds",
					Args: []string{
						fmt.Sprintf("-port=%d", ldsPort),
						"-listener-config=/etc/cf-assets/envoy_config/listeners.json",
					},
					Env: []string{},
					Limits: garden.ResourceLimits{
						Nofile: proto.Uint64(1024),
					},
					Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
					BindMounts: []garden.BindMount{
						{
							SrcPath: "/some/source",
							DstPath: "/some/destintation",
							Mode:    garden.BindMountModeRO,
							Origin:  garden.BindMountOriginHost,
						},
					},
				}))
			})

			Context("when the container is privileged", func() {
				BeforeEach(func() {
					container.Privileged = true
				})

				It("runs the listener discovery service (lds) in the same container", func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					specs := []garden.ProcessSpec{}
					for i := 0; i < gardenContainer.RunCallCount(); i++ {
						spec, _ := gardenContainer.RunArgsForCall(i)
						specs = append(specs, spec)
					}

					Expect(specs).NotTo(ContainElement(garden.ProcessSpec{
						ID:   "",
						Path: "/etc/cf-assets/envoy/lds",
						Args: []string{
							fmt.Sprintf("-port=%d", ldsPort),
							"-listener-config=/etc/cf-assets/envoy_config/listeners.json",
						},
						Env: []string{},
						Limits: garden.ResourceLimits{
							Nofile: proto.Uint64(1024),
						},
					}))
				})
			})

			Context("when the container proxy is disabled on the container", func() {
				BeforeEach(func() {
					container.EnableContainerProxy = false
				})

				It("does not run the container proxy", func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(1))
					paths := []string{}
					for i := 0; i < gardenContainer.RunCallCount(); i++ {
						spec, _ := gardenContainer.RunArgsForCall(i)
						paths = append(paths, spec.Path)
					}

					Expect(paths).NotTo(ContainElement("sh"))
				})
			})
		})

		Describe("declarative healthchecks", func() {
			var (
				process                       ifrit.Process
				readinessProcess              *gardenfakes.FakeProcess
				readinessCh                   chan int
				livenessProcess               *gardenfakes.FakeProcess
				livenessCh                    chan int
				actionProcess                 *gardenfakes.FakeProcess
				actionCh                      chan int
				monitorProcess                *gardenfakes.FakeProcess
				monitorCh                     chan int
				readinessIO                   chan garden.ProcessIO
				livenessIO                    chan garden.ProcessIO
				processLock                   sync.Mutex
				specs                         chan garden.ProcessSpec
				declarativeHealthcheckSrcPath string = filepath.Join(string(os.PathSeparator), "dir", "healthcheck")
			)

			BeforeEach(func() {
				// get rid of race condition caused by read inside the RunStub
				processLock.Lock()
				defer processLock.Unlock()

				readinessIO = make(chan garden.ProcessIO, 1)
				livenessIO = make(chan garden.ProcessIO, 1)
				specs = make(chan garden.ProcessSpec, 10)
				// make the race detector happy
				readinessIOCh := readinessIO
				livenessIOCh := livenessIO
				specsCh := specs

				readinessCh = make(chan int)
				readinessProcess = makeProcess(readinessCh)

				livenessCh = make(chan int, 1)
				livenessProcess = makeProcess(livenessCh)

				actionCh = make(chan int, 1)
				actionProcess = makeProcess(actionCh)

				monitorCh = make(chan int)
				monitorProcess = makeProcess(monitorCh)

				healthcheckCallCount := int64(0)
				gardenContainer.RunStub = func(spec garden.ProcessSpec, io garden.ProcessIO) (process garden.Process, err error) {
					specsCh <- spec

					defer GinkgoRecover()
					// get rid of race condition caused by write inside the BeforeEach

					processLock.Lock()
					defer processLock.Unlock()

					switch spec.Path {
					case "/action/path":
						return actionProcess, nil
					case filepath.Join(transformer.HealthCheckDstPath, "healthcheck"):
						oldCount := atomic.AddInt64(&healthcheckCallCount, 1)
						switch oldCount {
						case 1:
							readinessIOCh <- io
							return readinessProcess, nil
						case 2:
							livenessIOCh <- io
							return livenessProcess, nil
						}
					case "/monitor/path":
						return monitorProcess, nil
					}

					err = errors.New("")
					Fail("unexpected executable path: " + spec.Path)
					return
				}
				container = executor.Container{
					RunInfo: executor.RunInfo{
						StartTimeoutMs: 1000,
						Action: &models.Action{
							RunAction: &models.RunAction{
								Path: "/action/path",
							},
						},
						Monitor: &models.Action{
							RunAction: &models.RunAction{
								Path: "/monitor/path",
							},
						},
						CheckDefinition: &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             5432,
										RequestTimeoutMs: 100,
										Path:             "/some/path",
									},
								},
							},
						},
					},
				}
			})

			JustBeforeEach(func() {
				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())

				process = ifrit.Background(runner)
			})

			AfterEach(func() {
				close(readinessCh)
				livenessCh <- 1 // the healthcheck in liveness mode can only exit by failing
				close(actionCh)
				close(monitorCh)
				ginkgomon.Interrupt(process)
			})

			Context("when they are enabled", func() {
				BeforeEach(func() {
					options = append(options, transformer.WithDeclarativeHealthchecks())

					container.StartTimeoutMs = 1000
				})

				AfterEach(func() {
					process.Signal(os.Kill)
				})

				Context("and no check definitions exist", func() {
					BeforeEach(func() {
						container.CheckDefinition = nil
					})

					JustBeforeEach(func() {
						clock.WaitForWatcherAndIncrement(unhealthyMonitoringInterval)
					})

					It("uses the monitor action", func() {
						Eventually(gardenContainer.RunCallCount, 5*time.Second).Should(Equal(2))
						paths := []string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							paths = append(paths, spec.Path)
						}

						Expect(paths).To(ContainElement("/monitor/path"))
					})

					Context("and container proxy is enabled", func() {
						BeforeEach(func() {
							options = append(options, transformer.WithContainerProxy(time.Second))
							cfg.BindMounts = append(cfg.BindMounts, garden.BindMount{
								Origin:  garden.BindMountOriginHost,
								SrcPath: declarativeHealthcheckSrcPath,
								DstPath: transformer.HealthCheckDstPath,
							})
							cfg.ProxyTLSPorts = []uint16{61001}
						})

						It("runs healthchecks for the envoy proxy ports", func() {
							Eventually(specs).Should(Receive(Equal(garden.ProcessSpec{
								ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "envoy-readiness-healthcheck-0"),
								Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
								Args: []string{
									"-port=61001",
									"-timeout=1000ms",
									fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
									fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
								},
								Env: []string{
									"CF_INSTANCE_IP=",
									"CF_INSTANCE_INTERNAL_IP=",
									"CF_INSTANCE_PORT=",
									"CF_INSTANCE_ADDR=",
									"CF_INSTANCE_PORTS=[]",
								},
								Limits: garden.ResourceLimits{
									Nofile: proto.Uint64(1024),
								},
								OverrideContainerLimits: &garden.ProcessLimits{},
								Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
								BindMounts: []garden.BindMount{
									{
										SrcPath: "/some/source",
										DstPath: "/some/destintation",
										Mode:    garden.BindMountModeRO,
										Origin:  garden.BindMountOriginHost,
									},
									{
										Origin:  garden.BindMountOriginHost,
										SrcPath: declarativeHealthcheckSrcPath,
										DstPath: transformer.HealthCheckDstPath,
									},
								},
							})))
						})
					})
				})

				Context("and an http check definition exists", func() {
					BeforeEach(func() {
						cfg.BindMounts = append(cfg.BindMounts, garden.BindMount{
							Origin:  garden.BindMountOriginHost,
							SrcPath: declarativeHealthcheckSrcPath,
							DstPath: transformer.HealthCheckDstPath,
						})
						container.CheckDefinition = &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             5432,
										RequestTimeoutMs: 100,
										Path:             "/some/path",
									},
								},
							},
						}
					})

					Context("and container proxy is enabled", func() {
						BeforeEach(func() {
							options = append(options, transformer.WithContainerProxy(time.Second))
							cfg.ProxyTLSPorts = []uint16{61001}
						})

						It("runs healthchecks for the envoy proxy ports", func() {
							Eventually(specs).Should(Receive(Equal(garden.ProcessSpec{
								ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "envoy-readiness-healthcheck-0"),
								Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
								Args: []string{
									"-port=61001",
									"-timeout=1000ms",
									fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
									fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
								},
								Env: []string{
									"CF_INSTANCE_IP=",
									"CF_INSTANCE_INTERNAL_IP=",
									"CF_INSTANCE_PORT=",
									"CF_INSTANCE_ADDR=",
									"CF_INSTANCE_PORTS=[]",
								},
								Limits: garden.ResourceLimits{
									Nofile: proto.Uint64(1024),
								},
								OverrideContainerLimits: &garden.ProcessLimits{},
								Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
								BindMounts: []garden.BindMount{
									{
										SrcPath: "/some/source",
										DstPath: "/some/destintation",
										Mode:    garden.BindMountModeRO,
										Origin:  garden.BindMountOriginHost,
									},
									{
										Origin:  garden.BindMountOriginHost,
										SrcPath: declarativeHealthcheckSrcPath,
										DstPath: transformer.HealthCheckDstPath,
									},
								},
							})))
						})
					})

					It("runs the readiness healthcheck in a sidecar container", func() {
						Eventually(specs).Should(Receive(Equal(garden.ProcessSpec{
							ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-0"),
							Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
							Args: []string{
								"-port=5432",
								"-timeout=100ms",
								"-uri=/some/path",
								fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
								fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
							},
							Env: []string{
								"CF_INSTANCE_IP=",
								"CF_INSTANCE_INTERNAL_IP=",
								"CF_INSTANCE_PORT=",
								"CF_INSTANCE_ADDR=",
								"CF_INSTANCE_PORTS=[]",
							},
							Limits: garden.ResourceLimits{
								Nofile: proto.Uint64(1024),
							},
							OverrideContainerLimits: &garden.ProcessLimits{},
							Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
							BindMounts: []garden.BindMount{
								{
									SrcPath: "/some/source",
									DstPath: "/some/destintation",
									Mode:    garden.BindMountModeRO,
									Origin:  garden.BindMountOriginHost,
								},
								{
									Origin:  garden.BindMountOriginHost,
									SrcPath: declarativeHealthcheckSrcPath,
									DstPath: transformer.HealthCheckDstPath,
								},
							},
						})))
					})

					Context("when the container is privileged", func() {
						BeforeEach(func() {
							container.Privileged = true
						})

						It("runs the healthcheck in a sidecar container", func() {
							Eventually(specs).Should(Receive(Equal(garden.ProcessSpec{
								ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-0"),
								Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
								Args: []string{
									"-port=5432",
									"-timeout=100ms",
									"-uri=/some/path",
									fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
									fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
								},
								Env: []string{
									"CF_INSTANCE_IP=",
									"CF_INSTANCE_INTERNAL_IP=",
									"CF_INSTANCE_PORT=",
									"CF_INSTANCE_ADDR=",
									"CF_INSTANCE_PORTS=[]",
								},
								Limits: garden.ResourceLimits{
									Nofile: proto.Uint64(1024),
								},
								OverrideContainerLimits: &garden.ProcessLimits{},
								Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
								BindMounts: []garden.BindMount{
									{
										SrcPath: "/some/source",
										DstPath: "/some/destintation",
										Mode:    garden.BindMountModeRO,
										Origin:  garden.BindMountOriginHost,
									},
									{
										Origin:  garden.BindMountOriginHost,
										SrcPath: declarativeHealthcheckSrcPath,
										DstPath: transformer.HealthCheckDstPath,
									},
								},
							})))
						})
					})

					Context("and the starttimeout is set to 0", func() {
						BeforeEach(func() {
							container.StartTimeoutMs = 0
						})

						It("runs the healthcheck with readiness timeout set to 0", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(2))
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=100ms",
								"-uri=/some/path",
								"-readiness-interval=1ms",
								"-readiness-timeout=0s",
							}))
						})
					})

					Context("and optional fields are missing", func() {
						BeforeEach(func() {
							container.CheckDefinition = &models.CheckDefinition{
								Checks: []*models.Check{
									&models.Check{
										HttpCheck: &models.HTTPCheck{
											Port: 5432,
										},
									},
								},
							}
						})

						It("uses sane defaults", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(2))
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=1000ms",
								"-uri=/",
								"-readiness-interval=1ms",
								"-readiness-timeout=1s",
							}))
						})
					})

					It("uses the check definition", func() {
						Eventually(gardenContainer.RunCallCount).Should(Equal(2))
						paths := []string{}
						args := [][]string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							paths = append(paths, spec.Path)
							args = append(args, spec.Args)
						}

						Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
						Expect(args).To(ContainElement([]string{
							"-port=5432",
							"-timeout=100ms",
							"-uri=/some/path",
							"-readiness-interval=1ms", // 1ms
							"-readiness-timeout=1s",
						}))
					})

					Context("when the readiness check times out", func() {
						JustBeforeEach(func() {
							By("waiting for the action and readiness check processes to start")
							var io garden.ProcessIO
							Eventually(readinessIO).Should(Receive(&io))
							_, err := io.Stdout.Write([]byte("readiness check failed"))
							Expect(err).NotTo(HaveOccurred())

							By("timing out the readiness check")
							Eventually(gardenContainer.RunCallCount).Should(Equal(2))

							Consistently(readinessProcess.SignalCallCount).Should(Equal(0))
							readinessCh <- 1
							Eventually(actionProcess.SignalCallCount).Should(Equal(1))
							actionCh <- 2
						})

						It("suppress the readiness check log", func() {
							Eventually(process.Wait()).Should(Receive(HaveOccurred()))
							Consistently(fakeMetronClient.SendAppLogCallCount).Should(Equal(2))
							_, msg0, _, _ := fakeMetronClient.SendAppLogArgsForCall(0)
							_, msg1, _, _ := fakeMetronClient.SendAppLogArgsForCall(1)
							Expect([]string{msg0, msg1}).To(ConsistOf("Starting health monitoring of container", "Exit status 2"))
						})

						It("logs the readiness check output on stderr", func() {
							Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(Equal(2))
							logLines := map[string]string{}
							_, msg, source, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
							logLines[source] = msg
							_, msg, source, _ = fakeMetronClient.SendAppErrorLogArgsForCall(1)
							logLines[source] = msg
							Expect(logLines).To(Equal(map[string]string{
								"HEALTH": "readiness check failed",
								"test":   "Timed out after 1s: health check never passed.",
							}))
						})

						It("returns the readiness check output in the error", func() {
							Consistently(process.Ready()).ShouldNot(BeClosed())
						})

						It("returns the readiness check output in the error", func() {
							Eventually(process.Wait()).Should(Receive(MatchError(ContainSubstring("Instance never healthy after 1s: readiness check failed"))))
						})
					})

					Context("when the readiness check passes", func() {
						JustBeforeEach(func() {
							readinessCh <- 0
						})

						It("starts the liveness check", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(3))
							ids := []string{}
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								ids = append(ids, spec.ID)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "liveness-healthcheck-0")))
							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=100ms",
								"-uri=/some/path",
								"-liveness-interval=1s",
							}))
						})

						Context("when the liveness check exits", func() {
							JustBeforeEach(func() {
								Eventually(gardenContainer.RunCallCount).Should(Equal(3))

								By("waiting the action and liveness check processes to start")
								var io garden.ProcessIO
								Eventually(livenessIO).Should(Receive(&io))
								_, err := io.Stdout.Write([]byte("liveness check failed"))
								Expect(err).NotTo(HaveOccurred())

								By("exiting the liveness check")
								livenessCh <- 1
								Eventually(actionProcess.SignalCallCount).Should(Equal(1))
								actionCh <- 2
							})

							It("logs the liveness check output on stderr", func() {
								Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(Equal(1))
								logLines := map[string]string{}
								_, msg, source, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
								logLines[source] = msg
								Expect(logLines).To(Equal(map[string]string{
									"HEALTH": "liveness check failed",
								}))
							})

							It("returns the liveness check output in the error", func() {
								Eventually(process.Wait()).Should(Receive(MatchError(ContainSubstring("Instance became unhealthy: liveness check failed"))))
							})
						})
					})
				})

				Context("and a tcp check definition exists", func() {
					BeforeEach(func() {
						container.CheckDefinition = &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									TcpCheck: &models.TCPCheck{
										Port:             5432,
										ConnectTimeoutMs: 100,
									},
								},
							},
						}
					})

					Context("and optional fields are missing", func() {
						BeforeEach(func() {
							container.CheckDefinition = &models.CheckDefinition{
								Checks: []*models.Check{
									&models.Check{
										TcpCheck: &models.TCPCheck{
											Port: 5432,
										},
									},
								},
							}
						})

						It("uses sane defaults", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(2))
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=1000ms",
								"-readiness-interval=1ms",
								"-readiness-timeout=1s",
							}))
						})
					})

					It("uses the readiness check definition", func() {
						Eventually(gardenContainer.RunCallCount).Should(Equal(2))
						ids := []string{}
						paths := []string{}
						args := [][]string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							ids = append(ids, spec.ID)
							paths = append(paths, spec.Path)
							args = append(args, spec.Args)
						}

						Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-0")))
						Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
						Expect(args).To(ContainElement([]string{
							"-port=5432",
							"-timeout=100ms",
							"-readiness-interval=1ms",
							"-readiness-timeout=1s",
						}))
					})

					Context("when the readiness check passes", func() {
						JustBeforeEach(func() {
							readinessCh <- 0
						})

						It("uses the liveness check definition", func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(3))
							ids := []string{}
							paths := []string{}
							args := [][]string{}
							for i := 0; i < gardenContainer.RunCallCount(); i++ {
								spec, _ := gardenContainer.RunArgsForCall(i)
								ids = append(ids, spec.ID)
								paths = append(paths, spec.Path)
								args = append(args, spec.Args)
							}

							Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "liveness-healthcheck-0")))
							Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
							Expect(args).To(ContainElement([]string{
								"-port=5432",
								"-timeout=100ms",
								"-liveness-interval=1s",
							}))

						})
					})
				})

				Context("logs", func() {
					BeforeEach(func() {
						container.CheckDefinition = &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             5432,
										RequestTimeoutMs: 2000,
										Path:             "/some/path",
									},
								},
							},
						}
					})

					JustBeforeEach(func() {
						Eventually(fakeMetronClient.SendAppLogCallCount, 5).Should(BeNumerically(">=", 1))

						_, message, sourceName, _ := fakeMetronClient.SendAppLogArgsForCall(0)
						Expect(message).To(Equal("Starting health monitoring of container"))
						Expect(sourceName).To(Equal("test"))

						var io garden.ProcessIO
						Eventually(readinessIO).Should(Receive(&io))
						io.Stdout.Write([]byte("failed"))

						readinessCh <- 1
					})

					It("should default to HEALTH for log source", func() {
						Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(BeNumerically(">=", 1))
						_, _, sourceName, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
						Expect(sourceName).To(Equal("HEALTH"))
					})

					Context("when log source defined", func() {
						BeforeEach(func() {
							container.CheckDefinition.LogSource = "healthcheck"
						})

						It("logs healthcheck errors with log source from check defintion", func() {
							Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(BeNumerically(">=", 1))
							_, _, sourceName, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
							Expect(sourceName).To(Equal("healthcheck"))
						})
					})
				})

				Context("and multiple check definitions exists", func() {
					var (
						otherReadinessProcess *gardenfakes.FakeProcess
						otherReadinessCh      chan int
						otherLivenessProcess  *gardenfakes.FakeProcess
						otherLivenessCh       chan int
					)

					BeforeEach(func() {
						// get rid of race condition caused by read inside the RunStub
						processLock.Lock()
						defer processLock.Unlock()

						otherReadinessCh = make(chan int)
						otherReadinessProcess = makeProcess(otherReadinessCh)

						otherLivenessCh = make(chan int)
						otherLivenessProcess = makeProcess(otherLivenessCh)

						healthcheckCallCount := int64(0)
						gardenContainer.RunStub = func(spec garden.ProcessSpec, io garden.ProcessIO) (process garden.Process, err error) {
							defer GinkgoRecover()
							// get rid of race condition caused by write inside the BeforeEach
							processLock.Lock()
							defer processLock.Unlock()

							switch spec.Path {
							case "/action/path":
								return actionProcess, nil
							case filepath.Join(transformer.HealthCheckDstPath, "healthcheck"):
								oldCount := atomic.AddInt64(&healthcheckCallCount, 1)
								switch oldCount {
								case 1:
									return readinessProcess, nil
								case 2:
									return otherReadinessProcess, nil
								case 3:
									return livenessProcess, nil
								case 4:
									return otherLivenessProcess, nil
								}
								return livenessProcess, nil
							case "/monitor/path":
								return monitorProcess, nil
							}

							err = errors.New("")
							Fail("unexpected executable path: " + spec.Path)
							return
						}

						container.CheckDefinition = &models.CheckDefinition{
							Checks: []*models.Check{
								&models.Check{
									TcpCheck: &models.TCPCheck{
										Port:             2222,
										ConnectTimeoutMs: 100,
									},
								},
								&models.Check{
									HttpCheck: &models.HTTPCheck{
										Port:             8080,
										RequestTimeoutMs: 100,
									},
								},
							},
						}
					})

					AfterEach(func() {
						close(otherReadinessCh)
						close(otherLivenessCh)
					})

					It("uses the check definition instead of the monitor action", func() {
						Eventually(gardenContainer.RunCallCount).Should(Equal(3))
						ids := []string{}
						paths := []string{}
						args := [][]string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							ids = append(ids, spec.ID)
							paths = append(paths, spec.Path)
							args = append(args, spec.Args)
						}

						Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-0")))
						Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "readiness-healthcheck-1")))

						Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
						Expect(args).To(ContainElement([]string{
							"-port=2222",
							"-timeout=100ms",
							"-readiness-interval=1ms",
							"-readiness-timeout=1s",
						}))
						Expect(args).To(ContainElement([]string{
							"-port=8080",
							"-timeout=100ms",
							"-uri=/",
							"-readiness-interval=1ms",
							"-readiness-timeout=1s",
						}))
					})

					Context("when one of the readiness checks finish", func() {
						JustBeforeEach(func() {
							Eventually(gardenContainer.RunCallCount).Should(Equal(3))
							readinessCh <- 0
						})

						It("waits for both healthchecks to pass", func() {
							Consistently(gardenContainer.RunCallCount).Should(Equal(3))
						})

						Context("and the other readiness check finish", func() {
							JustBeforeEach(func() {
								otherReadinessCh <- 0
							})

							It("starts the liveness checks", func() {
								Eventually(gardenContainer.RunCallCount).Should(Equal(5))
								ids := []string{}
								paths := []string{}
								args := [][]string{}
								for i := 0; i < gardenContainer.RunCallCount(); i++ {
									spec, _ := gardenContainer.RunArgsForCall(i)
									ids = append(ids, spec.ID)
									paths = append(paths, spec.Path)
									args = append(args, spec.Args)
								}

								Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "liveness-healthcheck-0")))
								Expect(ids).To(ContainElement(fmt.Sprintf("%s-%s", gardenContainer.Handle(), "liveness-healthcheck-1")))

								Expect(paths).To(ContainElement(filepath.Join(transformer.HealthCheckDstPath, "healthcheck")))
								Expect(args).To(ContainElement([]string{
									"-port=2222",
									"-timeout=100ms",
									"-liveness-interval=1s",
								}))
								Expect(args).To(ContainElement([]string{
									"-port=8080",
									"-timeout=100ms",
									"-uri=/",
									"-liveness-interval=1s",
								}))
							})

							Context("when either liveness check exit", func() {
								JustBeforeEach(func() {
									Eventually(gardenContainer.RunCallCount).Should(Equal(5))
									livenessCh <- 1
								})

								It("signals the process and exit", func() {
									Eventually(otherLivenessProcess.SignalCallCount).ShouldNot(BeZero())
									otherLivenessCh <- 1

									Eventually(actionProcess.SignalCallCount).ShouldNot(BeZero())
									actionCh <- 0

									Eventually(process.Wait()).Should(Receive(HaveOccurred()))
								})
							})
						})
					})
				})
			})

			Context("when they are disabled", func() {
				It("ignores the check definition and use the MonitorAction", func() {
					clock.WaitForWatcherAndIncrement(unhealthyMonitoringInterval)
					Eventually(gardenContainer.RunCallCount).Should(Equal(2))
					paths := []string{}
					for i := 0; i < gardenContainer.RunCallCount(); i++ {
						spec, _ := gardenContainer.RunArgsForCall(i)
						paths = append(paths, spec.Path)
					}

					Expect(paths).To(ContainElement("/monitor/path"))
				})

				Context("and container proxy is enabled", func() {
					BeforeEach(func() {
						options = append(options, transformer.WithContainerProxy(time.Second))
						cfg.BindMounts = append(cfg.BindMounts, garden.BindMount{
							Origin:  garden.BindMountOriginHost,
							SrcPath: declarativeHealthcheckSrcPath,
							DstPath: transformer.HealthCheckDstPath,
						})
						cfg.ProxyTLSPorts = []uint16{61001}
					})

					It("does not run healthchecks for the envoy proxy ports", func() {
						Consistently(specs).ShouldNot(Receive(Equal(garden.ProcessSpec{
							ID:   fmt.Sprintf("%s-%s", gardenContainer.Handle(), "envoy-readiness-healthcheck-0"),
							Path: filepath.Join(transformer.HealthCheckDstPath, "healthcheck"),
							Args: []string{
								"-port=61001",
								"-timeout=1000ms",
								fmt.Sprintf("-readiness-interval=%s", unhealthyMonitoringInterval),
								fmt.Sprintf("-readiness-timeout=%s", 1000*time.Millisecond),
							},
							Env: []string{},
							Limits: garden.ResourceLimits{
								Nofile: proto.Uint64(1024),
							},
							OverrideContainerLimits: &garden.ProcessLimits{},
							Image: garden.ImageRef{URI: "preloaded:cflinuxfs2"},
							BindMounts: []garden.BindMount{
								{
									SrcPath: "/some/source",
									DstPath: "/some/destintation",
									Mode:    garden.BindMountModeRO,
									Origin:  garden.BindMountOriginHost,
								},
								{
									Origin:  garden.BindMountOriginHost,
									SrcPath: declarativeHealthcheckSrcPath,
									DstPath: transformer.HealthCheckDstPath,
								},
							},
						})))
					})
				})

				Context("and there is no monitor action", func() {
					BeforeEach(func() {
						container.Monitor = nil
					})

					It("does not run any healthchecks", func() {
						Eventually(gardenContainer.RunCallCount).Should(Equal(1))
						Consistently(gardenContainer.RunCallCount).Should(Equal(1))

						paths := []string{}
						for i := 0; i < gardenContainer.RunCallCount(); i++ {
							spec, _ := gardenContainer.RunArgsForCall(i)
							paths = append(paths, spec.Path)
						}

						Expect(paths).To(ContainElement("/action/path"))
					})
				})
			})
		})

		Context("when there is no setup", func() {
			BeforeEach(func() {
				container.Setup = nil
			})

			It("returns a codependent step for the action/monitor", func() {
				gardenContainer.RunReturns(&gardenfakes.FakeProcess{}, nil)

				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())

				process := ifrit.Background(runner)

				Eventually(gardenContainer.RunCallCount).Should(Equal(1))
				processSpec, _ := gardenContainer.RunArgsForCall(0)
				Expect(processSpec.Path).To(Equal("/action/path"))
				Consistently(gardenContainer.RunCallCount).Should(Equal(1))

				clock.Increment(1 * time.Second)
				Eventually(gardenContainer.RunCallCount).Should(Equal(2))
				processSpec, _ = gardenContainer.RunArgsForCall(1)
				Expect(processSpec.Path).To(Equal("/monitor/path"))
				Eventually(process.Ready()).Should(BeClosed())

				process.Signal(os.Interrupt)
				clock.Increment(1 * time.Second)
				Eventually(process.Wait()).Should(Receive(nil))
			})
		})

		Context("when there is no monitor", func() {
			BeforeEach(func() {
				container.Monitor = nil
				container.Setup = nil
			})

			It("does not run the monitor step and immediately says the healthcheck passed", func() {
				blockCh := make(chan struct{})
				defer close(blockCh)

				fakeGardenProcess := &gardenfakes.FakeProcess{}
				fakeGardenProcess.WaitStub = func() (int, error) {
					<-blockCh
					return 0, nil
				}
				gardenContainer.RunReturns(fakeGardenProcess, nil)

				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())

				process := ifrit.Background(runner)
				Eventually(process.Ready()).Should(BeClosed())

				Eventually(gardenContainer.RunCallCount).Should(Equal(1))
				processSpec, _ := gardenContainer.RunArgsForCall(0)
				Expect(processSpec.Path).To(Equal("/action/path"))
				Consistently(gardenContainer.RunCallCount).Should(Equal(1))
			})
		})

		Context("MonitorAction", func() {
			var (
				process ifrit.Process
			)

			JustBeforeEach(func() {
				runner, err := optimusPrime.StepsRunner(logger, container, gardenContainer, logStreamer, cfg)
				Expect(err).NotTo(HaveOccurred())
				process = ifrit.Background(runner)
			})

			AfterEach(func() {
				ginkgomon.Interrupt(process)
			})

			BeforeEach(func() {
				container.Setup = nil
				container.Monitor = &models.Action{
					ParallelAction: models.Parallel(
						&models.RunAction{
							Path:              "/monitor/path",
							SuppressLogOutput: true,
						},
						&models.RunAction{
							Path:              "/monitor/path",
							SuppressLogOutput: true,
						},
					),
				}
			})

			Context("SuppressLogOutput", func() {
				var (
					monitorCh, actionCh chan int
				)

				BeforeEach(func() {
					monitorCh = make(chan int, 2)
					actionCh = make(chan int, 1)

					gardenContainer.RunStub = func(processSpec garden.ProcessSpec, processIO garden.ProcessIO) (garden.Process, error) {
						switch processSpec.Path {
						case "/monitor/path":
							return makeProcess(monitorCh), nil
						case "/action/path":
							return makeProcess(actionCh), nil
						default:
							return &gardenfakes.FakeProcess{}, nil
						}
					}
				})

				AfterEach(func() {
					close(monitorCh)
					close(actionCh)
				})

				JustBeforeEach(func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(1))
					clock.Increment(1 * time.Second)
					Eventually(gardenContainer.RunCallCount).Should(Equal(3))
				})

				It("is ignored", func() {
					processSpec, processIO := gardenContainer.RunArgsForCall(1)
					Expect(processSpec.Path).To(Equal("/monitor/path"))
					Expect(container.Monitor.RunAction.GetSuppressLogOutput()).Should(BeFalse())
					Expect(processIO.Stdout).ShouldNot(Equal(ioutil.Discard))
					monitorCh <- 0
					monitorCh <- 0
					Eventually(process.Ready()).Should(BeClosed())
				})
			})

			Context("logs", func() {
				var (
					exitStatusCh    chan int
					monitorProcess1 *gardenfakes.FakeProcess
					monitorProcess2 *gardenfakes.FakeProcess
				)

				BeforeEach(func() {
					monitorProcess1 = &gardenfakes.FakeProcess{}
					monitorProcess2 = &gardenfakes.FakeProcess{}
					actionProcess := &gardenfakes.FakeProcess{}
					exitStatusCh = make(chan int)
					actionProcess.WaitStub = func() (int, error) {
						return <-exitStatusCh, nil
					}

					monitorProcessChan1 := make(chan *garden.ProcessIO, 4)
					monitorProcessChan2 := make(chan *garden.ProcessIO, 4)

					monitorProcess1.WaitStub = func() (int, error) {
						procIO := <-monitorProcessChan1

						if monitorProcess1.WaitCallCount() == 2 {
							procIO.Stdout.Write([]byte("healthcheck failed"))
							return 1, nil
						}
						return 0, nil
					}

					monitorProcess2.WaitStub = func() (int, error) {
						procIO := <-monitorProcessChan2

						if monitorProcess2.WaitCallCount() == 2 {
							procIO.Stdout.Write([]byte("healthcheck failed"))
							return 1, nil
						}
						return 0, nil
					}

					monitorProcessRun := uint32(0)

					gardenContainer.RunStub = func(processSpec garden.ProcessSpec, processIO garden.ProcessIO) (garden.Process, error) {
						if processSpec.Path == "/monitor/path" {
							if atomic.AddUint32(&monitorProcessRun, 1)%2 == 0 {
								monitorProcessChan1 <- &processIO
								return monitorProcess1, nil
							}
							monitorProcessChan2 <- &processIO
							return monitorProcess2, nil
						} else if processSpec.Path == "/action/path" {
							return actionProcess, nil
						}
						return &gardenfakes.FakeProcess{}, nil
					}
				})

				AfterEach(func() {
					Eventually(exitStatusCh).Should(BeSent(1))
				})

				JustBeforeEach(func() {
					Eventually(gardenContainer.RunCallCount).Should(Equal(1))

					By("starting the readiness check")
					clock.WaitForWatcherAndIncrement(1 * time.Second)
					Eventually(gardenContainer.RunCallCount).Should(Equal(3))
					Eventually(monitorProcess1.WaitCallCount).Should(Equal(1))
					Eventually(monitorProcess2.WaitCallCount).Should(Equal(1))

					By("starting the liveness check")
					clock.WaitForWatcherAndIncrement(1 * time.Second)
					Eventually(gardenContainer.RunCallCount).Should(Equal(5))
					Eventually(monitorProcess1.WaitCallCount).Should(Equal(2))
					Eventually(monitorProcess2.WaitCallCount).Should(Equal(2))
				})

				It("logs healthcheck error with the same source in a readable way", func() {
					Eventually(fakeMetronClient.SendAppErrorLogCallCount).Should(Equal(1))
					_, message, sourceName, _ := fakeMetronClient.SendAppErrorLogArgsForCall(0)
					Expect(sourceName).To(Equal("test"))
					Expect(message).To(ContainSubstring("healthcheck failed; healthcheck failed"))
				})

				It("logs the container lifecycle", func() {
					Eventually(fakeMetronClient.SendAppLogCallCount).Should(Equal(3))
					_, message, _, _ := fakeMetronClient.SendAppLogArgsForCall(0)
					Expect(message).To(Equal("Starting health monitoring of container"))
					_, message, _, _ = fakeMetronClient.SendAppLogArgsForCall(1)
					Expect(message).To(Equal("Container became healthy"))
					_, message, _, _ = fakeMetronClient.SendAppLogArgsForCall(2)
					Expect(message).To(Equal("Container became unhealthy"))
				})
			})
		})
	})
})
