--------------------------------------------------------------------------

-- Healthcheck library for OpenResty.

--

-- Some notes on the usage of this library:

--

-- - Each target will have 4 counters, 1 success counter and 3 failure

-- counters ('http', 'tcp', and 'timeout'). Any failure will _only_ reset the

-- success counter, but a success will reset _all three_ failure counters.

--

-- - All targets are uniquely identified by their IP address and port number

-- combination, most functions take those as arguments.

--

-- - All keys in the SHM will be namespaced by the healthchecker name as

-- provided to the `new` function. Hence no collissions will occur on shm-keys

-- as long as the `name` is unique.

--

-- - Active healthchecks will be synchronized across workers, such that only

-- a single active healthcheck runs.

--

-- - Events will be raised in every worker, see [lua-resty-worker-events](https://github.com/Kong/lua-resty-worker-events)

-- for details.

--

-- @copyright 2017-2023 Kong Inc.

-- @author Hisham Muhammad, Thijs Schreijer

-- @license Apache 2.0

local ERR = ngx.ERR

local WARN = ngx.WARN

local DEBUG = ngx.DEBUG

local ngx_log = ngx.log

local tostring = tostring

local ipairs = ipairs

local table_insert = table.insert

local table_remove = table.remove

local table_concat = table.concat

local string_format = string.format

local math_max = math.max

local ssl = require("ngx.ssl")

local resty_timer = require "resty.timer"

local bit = require("bit")

local re_find = ngx.re.find

local ngx_now = ngx.now

local ngx_worker_id = ngx.worker.id

local ngx_worker_pid = ngx.worker.pid

local pcall = pcall

local get_phase = ngx.get_phase

local type = type

local assert = assert

local RESTY_WORKER_EVENTS_VER = "0.3.3"

local new_tab

local nkeys

local is_array

local codec

local TESTING = _G.__TESTING_HEALTHCHECKER or false

do

local ok

ok, new_tab = pcall(require, "table.new")

if not ok then

new_tab = function () return {} end

end

-- OpenResty branch of LuaJIT New API

ok, nkeys = pcall(require, "table.nkeys")

if not ok then

nkeys = function (tab)

local count = 0

for _, v in pairs(tab) do

if v ~= nil then

count = count + 1

end

end

return count

end

end

ok, is_array = pcall(require, "table.isarray")

if not ok then

is_array = function(t)

for k in pairs(t) do

if type(k) ~= "number" or math.floor(k) ~= k then

return false

end

end

return true

end

end

ok, codec = pcall(require, "string.buffer")

if not ok then

codec = require("cjson.safe").new()

end

end

local worker_events

--- This function loads the worker events module received as arg. It will throw

-- error() if it is not possible to load the module.

local function load_events_module(self)

if self.events_module == "resty.worker.events" then

worker_events = require("resty.worker.events")

assert(worker_events, "could not load lua-resty-worker-events")

assert(worker_events._VERSION == RESTY_WORKER_EVENTS_VER,

"unsupported lua-resty-worker-events version")

elseif self.events_module == "resty.events" then

worker_events = require("resty.events.compat")

else

error("unknown events module")

end

assert(worker_events.configured(), "please configure the '" ..

self.events_module .. "' module before using 'lua-resty-healthcheck'")

end

-- constants

local EVENT_SOURCE_PREFIX = "lua-resty-healthcheck"

local LOG_PREFIX = "[healthcheck] "

local SHM_PREFIX = "lua-resty-healthcheck:"

local EMPTY = setmetatable({},{

__newindex = function()

error("the EMPTY table is read only, check your code!", 2)

end

})

--- timer constants

-- evaluate active checks every 0.1s

local CHECK_INTERVAL = 0.1

-- use a 10% jitter to start each worker timer

local CHECK_JITTER = CHECK_INTERVAL * 0.1

-- lock valid period: the worker which acquires the lock owns it for 15 times

-- the check interval. If it does not update the shm during this period, we

-- consider that it is not able to continue checking (the worker probably was killed)

local LOCK_PERIOD = CHECK_INTERVAL * 15

-- interval between stale targets cleanup

local CLEANUP_INTERVAL = CHECK_INTERVAL * 25

-- Counters: a 32-bit shm integer can hold up to four 8-bit counters.

local CTR_SUCCESS = 0x00000001

local CTR_HTTP = 0x00000100

local CTR_TCP = 0x00010000

local CTR_TIMEOUT = 0x01000000

local MASK_FAILURE = 0xffffff00

local MASK_SUCCESS = 0x000000ff

local COUNTER_NAMES = {

[CTR_SUCCESS] = "SUCCESS",

[CTR_HTTP] = "HTTP",

[CTR_TCP] = "TCP",

[CTR_TIMEOUT] = "TIMEOUT",

}

--- The list of potential events generated.

-- The `checker.EVENT_SOURCE` field can be used to subscribe to the events, see the

-- example below. Each of the events will get a table passed containing

-- the target details `ip`, `port`, and `hostname`.

-- See [lua-resty-worker-events](https://github.com/Kong/lua-resty-worker-events).

-- @field remove Event raised when a target is removed from the checker.

-- @field healthy This event is raised when the target status changed to

-- healthy (and when a target is added as `healthy`).

-- @field unhealthy This event is raised when the target status changed to

-- unhealthy (and when a target is added as `unhealthy`).

-- @field mostly_healthy This event is raised when the target status is

-- still healthy but it started to receive "unhealthy" updates via active or

-- passive checks.

-- @field mostly_unhealthy This event is raised when the target status is

-- still unhealthy but it started to receive "healthy" updates via active or

-- passive checks.

-- @table checker.events

-- @usage -- Register for all events from `my_checker`

-- local event_callback = function(target, event, source, source_PID)

-- local t = target.ip .. ":" .. target.port .." by name '" ..

-- target.hostname .. "' ")

--

-- if event == my_checker.events.remove then

-- print(t .. "has been removed")

-- elseif event == my_checker.events.healthy then

-- print(t .. "is now healthy")

-- elseif event == my_checker.events.unhealthy then

-- print(t .. "is now unhealthy")

-- end

-- end

--

-- worker_events.register(event_callback, my_checker.EVENT_SOURCE)

local EVENTS = setmetatable({}, {

__index = function(self, key)

error(("'%s' is not a valid event name"):format(tostring(key)))

end

})

for _, event in ipairs({

"remove",

"healthy",

"unhealthy",

"mostly_healthy",

"mostly_unhealthy",

"clear",

}) do

EVENTS[event] = event

end

local INTERNAL_STATES = {}

for i, key in ipairs({

"healthy",

"unhealthy",

"mostly_healthy",

"mostly_unhealthy",

}) do

INTERNAL_STATES[i] = key

INTERNAL_STATES[key] = i

end

-- Some color for demo purposes

local use_color = false

local id = function(x) return x end

local worker_color = use_color and function(str) return ("\027["..tostring(31 + ngx_worker_pid() % 5).."m"..str.."\027[0m") end or id

-- Debug function

local function dump(...) print(require("pl.pretty").write({...})) end -- luacheck: ignore 211

local _M = {}

-- checker objects (weak) table

local hcs = setmetatable({}, {

__mode = "v",

})

local active_check_timer

local last_cleanup_check

-- serialize a table to a string

local serialize = codec.encode

-- deserialize a string to a table

local deserialize = codec.decode

local function key_for(key_prefix, ip, port, hostname)

return string_format("%s:%s:%s%s", key_prefix, ip, port, hostname and ":" .. hostname or "")

end

-- resty.lock timeout when yieldable

local LOCK_TIMEOUT = 5

local run_locked

do

-- resty_lock is restricted to this scope in order to keep sensitive

-- lock-handling code separate separate from all other business logic

--

-- If you need to use resty_lock in a way that is not covered by the

-- `run_locked` helper function defined below, it's strongly-advised to

-- define it fully within this scope unless you have a very good reason

--

-- (see https://github.com/Kong/lua-resty-healthcheck/pull/112)

local resty_lock = require "resty.lock"

local yieldable = {

rewrite = true,

access = true,

content = true,

timer = true,

}

local function run_in_timer(premature, self, key, fn, ...)

if premature then

return

end

local ok, err = run_locked(self, key, fn, ...)

if not ok then

self:log(ERR, "locked function for key '", key, "' failed in timer: ", err)

end

end

local function schedule(self, key, fn, ...)

local ok, err = ngx.timer.at(0, run_in_timer, self, key, fn, ...)

if not ok then

return nil, "failed scheduling locked function for key '" .. key ..

"', " .. err

end

return "scheduled"

end

-- resty.lock consumes these options immediately, so this table can be reused

local opts = {

exptime = 10, -- timeout after which lock is released anyway

timeout = LOCK_TIMEOUT, -- max wait time to acquire lock

}

---

-- Acquire a lock and run a function

--

-- The function call itself is wrapped with `pcall` to protect against

-- exception.

--

-- This function exhibits some special behavior when called during a

-- non-yieldable phase such as `init_worker` or `log`:

--

-- 1. The lock timeout is set to 0 to ensure that `resty.lock` does not

-- attempt to sleep/yield

-- 2. If acquiring the lock fails due to a timeout, `run_locked`

-- (this function) is re-scheduled to run in a timer. In this case,

-- the function returns `"scheduled"`

--

-- @param self The checker object

-- @param key the key/identifier to acquire a lock for

-- @param fn The function to execute

-- @param ... arguments that will be passed to fn

-- @return The results of the function; or nil and an error message

-- in case it fails locking.

function run_locked(self, key, fn, ...)

-- we're extra extra extra defensive in this code path

local typ = type(key)

-- XXX is a number key ever expected?

assert(typ == "string" or typ == "number",

"unexpected lock key type: " .. typ)

key = tostring(key)

-- first aqcuire a lock or conditionally re-schedule ourselves in a timer

local lock

do

local yield = yieldable[get_phase()]

if yield then

opts.timeout = LOCK_TIMEOUT

else

-- if yielding is not possible in the current phase, use a zero timeout

-- so that resty.lock will return `nil, "timeout"` immediately instead of

-- calling ngx.sleep()

opts.timeout = 0

end

local err

lock, err = resty_lock:new(self.shm_name, opts)

if not lock then

return nil, "failed creating lock for '" .. key .. "', " .. err

end

local elapsed

elapsed, err = lock:lock(key)

if not elapsed and err == "timeout" and not yield then

-- yielding is not possible in the current phase, so retry in a timer

return schedule(self, key, fn, ...)

elseif not elapsed then

return nil, "failed acquiring lock for '" .. key .. "', " .. err

end

end

local pok, perr, res = pcall(fn, ...)

local ok, err = lock:unlock()

if not ok then

self:log(ERR, "failed unlocking '", key, "', ", err)

end

if not pok then

return nil, "locked function threw an exception: " .. tostring(perr)

end

return perr, res

end

end

local checker = {}

------------------------------------------------------------------------------

-- Node management.

-- @section node-management

------------------------------------------------------------------------------

-- @return the target list from the shm, an empty table if not found, or

-- `nil + error` upon a failure

local function fetch_target_list(self)

local target_list, err = self.shm:get(self.TARGET_LIST)

if err then

return nil, "failed to fetch target_list from shm: " .. err

end

return target_list and deserialize(target_list) or {}

end

local function with_target_list(self, fn)

local targets, err = fetch_target_list(self)

if not targets then

return nil, err

end

-- this is only ever called in the context of `run_locked`,

-- so no pcall needed

return fn(targets)

end

--- Run the given function holding a lock on the target list.

-- @param self The checker object

-- @param fn The function to execute

-- @return The results of the function; or nil and an error message

-- in case it fails locking.

local function locking_target_list(self, fn)

local ok, err = run_locked(self, self.TARGET_LIST_LOCK, with_target_list, self, fn)

if ok == "scheduled" then

self:log(DEBUG, "target_list function re-scheduled in timer")

end

return ok, err

end

--- Get a target

local function get_target(self, ip, port, hostname)

hostname = hostname or ip

return ((self.targets[ip] or EMPTY)[port] or EMPTY)[hostname]

end

--- Add a target to the healthchecker.

-- When the ip + port + hostname combination already exists, it will simply

-- return success (without updating `is_healthy` status).

-- @param ip IP address of the target to check.

-- @param port the port to check against.

-- @param hostname (optional) hostname to set as the host header in the HTTP

-- probe request

-- @param is_healthy (optional) a boolean value indicating the initial state,

-- default is `true`.

-- @param hostheader (optional) a value to use for the Host header on

-- active healthchecks.

-- @return `true` on success, or `nil + error` on failure.

function checker:add_target(ip, port, hostname, is_healthy, hostheader)

ip = tostring(assert(ip, "no ip address provided"))

port = assert(tonumber(port), "no port number provided")

hostname = hostname or ip

if is_healthy == nil then

is_healthy = true

end

local internal_health = is_healthy and "healthy" or "unhealthy"

local ok, err = locking_target_list(self, function(target_list)

local found = false

-- check whether we already have this target

for _, target in ipairs(target_list) do

if target.ip == ip and target.port == port and target.hostname == (hostname) then

if target.purge_time == nil then

self:log(DEBUG, "adding an existing target: ", hostname or "", " ", ip,

":", port, " (ignoring)")

return false

end

target.purge_time = nil

found = true

internal_health = self:get_target_status(ip, port, hostname) and

"healthy" or "unhealthy"

break

end

end

-- we first add the internal health, and only then the updated list.

-- this prevents a state where a target is in the list, but does not

-- have a key in the shm.

local ok, err = self.shm:set(key_for(self.TARGET_STATE, ip, port, hostname),

INTERNAL_STATES[internal_health])

if not ok then

self:log(ERR, "failed to set initial health status in shm: ", err)

end

-- target does not exist, go add it

if not found then

target_list[#target_list + 1] = {

ip = ip,

port = port,

hostname = hostname,

hostheader = hostheader,

}

end

target_list = serialize(target_list)

ok, err = self.shm:set(self.TARGET_LIST, target_list)

if not ok then

return nil, "failed to store target_list in shm: " .. err

end

-- raise event for our newly added target

if not found then

self:raise_event(self.events[internal_health], ip, port, hostname)

end

return true

end)

if ok == false then

-- the target already existed, no event, but still success

return true

end

return ok, err

end

-- Remove health status entries from an individual target from shm

-- @param self The checker object

-- @param ip IP address of the target being checked.

-- @param port the port being checked against.

-- @param hostname hostname of the target being checked.

local function clear_target_data_from_shm(self, ip, port, hostname)

local ok, err = self.shm:set(key_for(self.TARGET_STATE, ip, port, hostname), nil)

if not ok then

self:log(ERR, "failed to remove health status from shm: ", err)

end

ok, err = self.shm:set(key_for(self.TARGET_COUNTER, ip, port, hostname), nil)

if not ok then

self:log(ERR, "failed to clear health counter from shm: ", err)

end

end

--- Remove a target from the healthchecker.

-- The target not existing is not considered an error.

-- @param ip IP address of the target being checked.

-- @param port the port being checked against.

-- @param hostname (optional) hostname of the target being checked.

-- @return `true` on success, or `nil + error` on failure.

function checker:remove_target(ip, port, hostname)

ip = tostring(assert(ip, "no ip address provided"))

port = assert(tonumber(port), "no port number provided")

return locking_target_list(self, function(target_list)

-- find the target

local target_found

for i, target in ipairs(target_list) do

if target.ip == ip and target.port == port and target.hostname == hostname then

target_found = target

table_remove(target_list, i)

break

end

end

if not target_found then

return true

end

-- go update the shm

target_list = serialize(target_list)

-- we first write the updated list, and only then remove the health

-- status; this prevents race conditions when a healthchecker gets the

-- initial state from the shm

local ok, err = self.shm:set(self.TARGET_LIST, target_list)

if not ok then

return nil, "failed to store target_list in shm: " .. err

end

clear_target_data_from_shm(self, ip, port, hostname)

-- raise event for our removed target

self:raise_event(self.events.remove, ip, port, hostname)

return true

end)

end

--- Clear all healthcheck data.

-- @return `true` on success, or `nil + error` on failure.

function checker:clear()

return locking_target_list(self, function(target_list)

local old_target_list = target_list

-- go update the shm

target_list = serialize({})

local ok, err = self.shm:set(self.TARGET_LIST, target_list)

if not ok then

return nil, "failed to store target_list in shm: " .. err

end

-- remove all individual statuses

for _, target in ipairs(old_target_list) do

local ip, port, hostname = target.ip, target.port, target.hostname

clear_target_data_from_shm(self, ip, port, hostname)

end

self.targets = {}

-- raise event for our removed target

self:raise_event(self.events.clear)

return true

end)

end

--- Clear all healthcheck data after a period of time.

-- Useful for keeping target status between configuration reloads.

-- @param delay delay in seconds before purging target state.

-- @return `true` on success, or `nil + error` on failure.

function checker:delayed_clear(delay)

assert(tonumber(delay), "no delay provided")

return locking_target_list(self, function(target_list)

local purge_time = ngx_now() + delay

-- add purge time to all targets

for _, target in ipairs(target_list) do

target.purge_time = purge_time

end

target_list = serialize(target_list)

local ok, err = self.shm:set(self.TARGET_LIST, target_list)

if not ok then

return nil, "failed to store target_list in shm: " .. err

end

return true

end)

end

--- Get the current status of the target.

-- @param ip IP address of the target being checked.

-- @param port the port being checked against.

-- @param hostname the hostname of the target being checked.

-- @return `true` if healthy, `false` if unhealthy, or `nil + error` on failure.

function checker:get_target_status(ip, port, hostname)

local target = get_target(self, ip, port, hostname)

if not target then

return nil, "target not found"

end

return target.internal_health == "healthy"

or target.internal_health == "mostly_healthy"

end

------------------------------------------------------------------------------

-- Health management.

-- Functions that allow reporting of failures/successes for passive checks.

-- @section health-management

------------------------------------------------------------------------------

-- Run the given function holding a lock on the target.

-- @param self The checker object

-- @param ip Target IP

-- @param port Target port

-- @param hostname Target hostname

-- @param fn The function to execute

-- @return The results of the function; or true in case it fails locking and

-- will retry asynchronously; or nil+err in case it fails to retry.

local function locking_target(self, ip, port, hostname, fn)

local key = key_for(self.TARGET_LOCK, ip, port, hostname)

local ok, err = run_locked(self, key, fn)

if ok == "scheduled" then

self:log(DEBUG, "target function for ", key, " was re-scheduled")

end

return ok, err

end

-- Extract the value of the counter at `idx` from multi-counter `multictr`.

-- @param multictr A 32-bit multi-counter holding 4 values.

-- @param idx The shift index specifying which counter to get.

-- @return The 8-bit value extracted from the 32-bit multi-counter.

local function ctr_get(multictr, idx)

return bit.band(multictr / idx, 0xff)

end

-- Increment the healthy or unhealthy counter. If the threshold of occurrences

-- is reached, it changes the status of the target in the shm and posts an

-- event.

-- @param self The checker object

-- @param health_report "healthy" for the success counter that drives a target

-- towards the healthy state; "unhealthy" for the failure counter.

-- @param ip Target IP

-- @param port Target port

-- @param hostname Target hostname

-- @param limit the limit after which target status is changed

-- @param ctr_type the counter to increment, see CTR_xxx constants

-- @return True if succeeded, or nil and an error message.

local function incr_counter(self, health_report, ip, port, hostname, limit, ctr_type)

-- fail fast on counters that are disabled by configuration

if limit == 0 then

return true

end

port = tonumber(port)

local target = get_target(self, ip, port, hostname)

if not target then

-- sync issue: warn, but return success

self:log(WARN, "trying to increment a target that is not in the list: ",

hostname and "(" .. hostname .. ") " or "", ip, ":", port)

return true

end

local current_health = target.internal_health

if health_report == current_health then

-- No need to count successes when internal health is fully "healthy"

-- or failures when internal health is fully "unhealthy"

return true

end

return locking_target(self, ip, port, hostname, function()

local counter_key = key_for(self.TARGET_COUNTER, ip, port, hostname)

local multictr, err = self.shm:incr(counter_key, ctr_type, 0)

if err then

return nil, err

end

local ctr = ctr_get(multictr, ctr_type)

self:log(WARN, health_report, " ", COUNTER_NAMES[ctr_type],

" increment (", ctr, "/", limit, ") for '", hostname or "",

"(", ip, ":", port, ")'")

local new_multictr

if ctr_type == CTR_SUCCESS then

new_multictr = bit.band(multictr, MASK_SUCCESS)

else

new_multictr = bit.band(multictr, MASK_FAILURE)

end

if new_multictr ~= multictr then

self.shm:set(counter_key, new_multictr)

end

local new_health

if ctr >= limit then

new_health = health_report

elseif current_health == "healthy" and bit.band(new_multictr, MASK_FAILURE) > 0 then

new_health = "mostly_healthy"

elseif current_health == "unhealthy" and bit.band(new_multictr, MASK_SUCCESS) > 0 then

new_health = "mostly_unhealthy"

end

if new_health and new_health ~= current_health then

local state_key = key_for(self.TARGET_STATE, ip, port, hostname)

self.shm:set(state_key, INTERNAL_STATES[new_health])

self:raise_event(self.events[new_health], ip, port, hostname)

end

return true

end)

end

--- Report a health failure.

-- Reports a health failure which will count against the number of occurrences

-- required to make a target "fall". The type of healthchecker,

-- "tcp" or "http" (see `new`) determines against which counter the occurence goes.

-- If `unhealthy.tcp_failures` (for TCP failures) or `unhealthy.http_failures`

-- is set to zero in the configuration, this function is a no-op

-- and returns `true`.

-- @param ip IP address of the target being checked.

-- @param port the port being checked against.

-- @param hostname (optional) hostname of the target being checked.

-- @param check (optional) the type of check, either "passive" or "active", default "passive".

-- @return `true` on success, or `nil + error` on failure.

function checker:report_failure(ip, port, hostname, check)

local checks = self.checks[check or "passive"]

local limit, ctr_type

if self.checks[check or "passive"].type == "tcp" then

limit = checks.unhealthy.tcp_failures

ctr_type = CTR_TCP

else

limit = checks.unhealthy.http_failures

ctr_type = CTR_HTTP

end

return incr_counter(self, "unhealthy", ip, port, hostname, limit, ctr_type)

end

--- Report a health success.

-- Reports a health success which will count against the number of occurrences

-- required to make a target "rise".

-- If `healthy.successes` is set to zero in the configuration,

-- this function is a no-op and returns `true`.

-- @param ip IP address of the target being checked.

-- @param port the port being checked against.

-- @param hostname (optional) hostname of the target being checked.

-- @param check (optional) the type of check, either "passive" or "active", default "passive".

-- @return `true` on success, or `nil + error` on failure.

function checker:report_success(ip, port, hostname, check)

local limit = self.checks[check or "passive"].healthy.successes

return incr_counter(self, "healthy", ip, port, hostname, limit, CTR_SUCCESS)

end

--- Report a http response code.

-- How the code is interpreted is based on the configuration for healthy and

-- unhealthy statuses. If it is in neither strategy, it will be ignored.

-- If `healthy.successes` (for healthy HTTP status codes)

-- or `unhealthy.http_failures` (fur unhealthy HTTP status codes)

-- is set to zero in the configuration, this function is a no-op

-- and returns `true`.

-- @param ip IP address of the target being checked.

-- @param port the port being checked against.

-- @param hostname (optional) hostname of the target being checked.

-- @param http_status the http statuscode, or nil to report an invalid http response.

-- @param check (optional) the type of check, either "passive" or "active", default "passive".

-- @return `true` on success, `nil` if the status was ignored (not in active or

-- passive health check lists) or `nil + error` on failure.

function checker:report_http_status(ip, port, hostname, http_status, check)

http_status = tonumber(http_status) or 0

local checks = self.checks[check or "passive"]

local status_type, limit, ctr

if checks.healthy.http_statuses[http_status] then

status_type = "healthy"

limit = checks.healthy.successes

ctr = CTR_SUCCESS

elseif checks.unhealthy.http_statuses[http_status]

or http_status == 0 then

status_type = "unhealthy"

limit = checks.unhealthy.http_failures

ctr = CTR_HTTP

else

return

end

return incr_counter(self, status_type, ip, port, hostname, limit, ctr)

end

--- Report a failure on TCP level.

-- If `unhealthy.tcp_failures` is set to zero in the configuration,

-- this function is a no-op and returns `true`.

-- @param ip IP address of the target being checked.

-- @param port the port being checked against.

-- @param hostname hostname of the target being checked.

-- @param operation The socket operation that failed:

-- "connect", "send" or "receive".

-- TODO check what kind of information we get from the OpenResty layer

-- in order to tell these error conditions apart

-- https://github.com/openresty/lua-resty-core/blob/master/lib/ngx/balancer.md#get_last_failure

-- @param check (optional) the type of check, either "passive" or "active", default "passive".

-- @return `true` on success, or `nil + error` on failure.

function checker:report_tcp_failure(ip, port, hostname, operation, check)

local limit = self.checks[check or "passive"].unhealthy.tcp_failures

-- TODO what do we do with the `operation` information

return incr_counter(self, "unhealthy", ip, port, hostname, limit, CTR_TCP)

end

--- Report a timeout failure.

-- If `unhealthy.timeouts` is set to zero in the configuration,

-- this function is a no-op and returns `true`.

-- @param ip IP address of the target being checked.

-- @param port the port being checked against.

-- @param hostname (optional) hostname of the target being checked.

-- @param check (optional) the type of check, either "passive" or "active", default "passive".

-- @return `true` on success, or `nil + error` on failure.

function checker:report_timeout(ip, port, hostname, check)

local limit = self.checks[check or "passive"].unhealthy.timeouts

return incr_counter(self, "unhealthy", ip, port, hostname, limit, CTR_TIMEOUT)

end

--- Sets the current status of all targets with the given hostname and port.

-- @param hostname hostname being checked.

-- @param port the port being checked against

-- @param is_healthy boolean: `true` for healthy, `false` for unhealthy

-- @return `true` on success, or `nil + error` on failure.

function checker:set_all_target_statuses_for_hostname(hostname, port, is_healthy)

assert(type(hostname) == "string", "no hostname provided")

port = assert(tonumber(port), "no port number provided")

assert(type(is_healthy) == "boolean")

local all_ok = true

local errs = {}

for _, target in ipairs(self.targets) do

if target.port == port and target.hostname == hostname then

local ok, err = self:set_target_status(target.ip, port, hostname, is_healthy)

if not ok then

all_ok = nil

table.insert(errs, err)

end

end

end

return all_ok, #errs > 0 and table_concat(errs, "; ") or nil

end

--- Sets the current status of the target.

-- This will immediately set the status and clear its counters.

-- @param ip IP address of the target being checked

-- @param port the port being checked against

-- @param hostname (optional) hostname of the target being checked.

-- @param is_healthy boolean: `true` for healthy, `false` for unhealthy

-- @return `true` on success, or `nil + error` on failure

function checker:set_target_status(ip, port, hostname, is_healthy)

ip = tostring(assert(ip, "no ip address provided"))

port = assert(tonumber(port), "no port number provided")

assert(type(is_healthy) == "boolean")

local health_report = is_healthy and "healthy" or "unhealthy"

local target = get_target(self, ip, port, hostname)

if not target then

-- sync issue: warn, but return success

self:log(WARN, "trying to set status for a target that is not in the list: ", ip, ":", port)

return true

end

local counter_key = key_for(self.TARGET_COUNTER, ip, port, hostname)

local state_key = key_for(self.TARGET_STATE, ip, port, hostname)

local ok, err = locking_target(self, ip, port, hostname, function()

local _, err = self.shm:set(counter_key, 0)

if err then

return nil, err

end

self.shm:set(state_key, INTERNAL_STATES[health_report])

if err then

return nil, err

end

self:raise_event(self.events[health_report], ip, port, hostname)

return true

end)

if ok then

self:log(WARN, health_report, " forced for ", hostname, " ", ip, ":", port)

end

return ok, err

end

-- Introspection function for testing

local function test_get_counter(self, ip, port, hostname)

return locking_target(self, ip, port, hostname, function()

local counter = self.shm:get(key_for(self.TARGET_COUNTER, ip, port, hostname))

local internal_health = (get_target(self, ip, port, hostname) or EMPTY).internal_health

return counter, internal_health

end)

end

--============================================================================

-- Healthcheck runner

--============================================================================

-- Builds and caches the serialized user-configured headers string for HTTP/1.x probes.

-- Uses ~= nil so that a cached empty string ("") is also a cache hit.

local function build_http_headers(self)

if self.checks.active._headers_str ~= nil then

return self.checks.active._headers_str

end

local req_headers = self.checks.active.headers

local headers

if req_headers == nil then

headers = ""

else