diff --git a/doc/configuration.txt b/doc/configuration.txt index 12a7eb7..994df07 100644 --- a/doc/configuration.txt +++ b/doc/configuration.txt @@ -2,9 +2,9 @@ HAProxy Configuration Manual ---------------------- - version 1.3.12.1 + version 1.3.12.3 willy tarreau - 2007/07/25 + 2007/09/11 This document covers the configuration language as implemented in the version @@ -47,6 +47,7 @@ The following keywords are supported in the "global" section : - nopoll - nosepoll - tune.maxpollevents + - spread-checks * Debugging - debug @@ -164,6 +165,13 @@ tune.maxpollevents latency at the expense of network bandwidth, and increasing it above 200 tends to trade latency for slightly increased bandwidth. +spread-checks <0..50, in percent> + Sometimes it is desirable to avoid sending health checks to servers at exact + intervals, for instance when many logical servers are located on the same + physical server. With the help of this parameter, it becomes possible to add + some randomness in the check interval between 0 and +/- 50%. A value between + 2 and 5 seems to show good results. The default value remains at 0. + 1.3) Debugging --------------- @@ -247,6 +255,7 @@ option httpchk X - X X option httpclose X X X X option httplog X X X X option logasap X X X - +option nolinger X X X X option persist X - X X option redispatch X - X X option smtpchk X - X X diff --git a/doc/haproxy-en.txt b/doc/haproxy-en.txt index 50a4802..3e0d4a7 100644 --- a/doc/haproxy-en.txt +++ b/doc/haproxy-en.txt @@ -1525,6 +1525,17 @@ options to enable TCP keep-alive : option clitcpka # enables keep-alive only on client side option srvtcpka # enables keep-alive only on server side +4.1.4) TCP lingering +-------------------- +It is possible to disable the system's lingering of data unacked by the client +at the end of a session. This is sometimes required when haproxy is used as a +front-end with lots of unreliable clients, and you observe thousands of sockets +in the FIN_WAIT state on the machine. This may be used in a frontend to affect +the client-side connection, as well as in a backend for the server-side +connection : + + option nolinger # disables data lingering + 4.2) Event logging ------------------ diff --git a/doc/haproxy-fr.txt b/doc/haproxy-fr.txt index 9845cff..19f8112 100644 --- a/doc/haproxy-fr.txt +++ b/doc/haproxy-fr.txt @@ -1562,6 +1562,17 @@ pour activer le maintien de session TCP : option clitcpka # active le keep-alive côté client option srvtcpka # active le keep-alive côté serveur +4.1.4) Rémanence des données TCP (lingering) +-------------------------------------------- +Il est possible de désactiver la conservation de données non acquittées par un +client à la fin d'une session. Cela peut parfois s'avérer nécessaire lorsque +haproxy est utilisé en face d'un grand nombre de clients non fiables et qu'un +nombre élevé de sockets en état FIN_WAIT est observé sur la machine. L'option +peut être utilisée dans un frontend pour ajuster les connexions vers les +clients, et dans un backend pour ajuster les connexions vers les serveurs : + + option nolinger # désactive la conservation de données + 4.2) Journalisation des connexions ---------------------------------- diff --git a/include/common/defaults.h b/include/common/defaults.h index 198288f..2c757d2 100644 --- a/include/common/defaults.h +++ b/include/common/defaults.h @@ -115,4 +115,14 @@ #define DEFAULT_MAXCONN SYSTEM_MAXCONN #endif +/* Minimum check interval for spread health checks. Servers with intervals + * greater than or equal to this value will have their checks spread apart + * and will be considered when searching the minimal interval. + * Others will be ignored for the minimal interval and will have their checks + * scheduled on a different basis. + */ +#ifndef SRV_CHK_INTER_THRES +#define SRV_CHK_INTER_THRES 1000 +#endif + #endif /* _COMMON_DEFAULTS_H */ diff --git a/include/proto/checks.h b/include/proto/checks.h index 839af55..8499175 100644 --- a/include/proto/checks.h +++ b/include/proto/checks.h @@ -26,6 +26,7 @@ #include void process_chk(struct task *t, struct timeval *next); +int start_checks(); #endif /* _PROTO_CHECKS_H */ diff --git a/include/types/backend.h b/include/types/backend.h index 57a8cb0..921eaed 100644 --- a/include/types/backend.h +++ b/include/types/backend.h @@ -60,6 +60,7 @@ #define PR_O_BALANCE_UH 0x10000000 /* balance on URI hash */ #define PR_O_BALANCE (PR_O_BALANCE_RR | PR_O_BALANCE_SH | PR_O_BALANCE_UH) #define PR_O_SMTP_CHK 0x20000000 /* use SMTP EHLO check for server health - pvandijk@vision6.com.au */ +#define PR_O_TCP_NOLING 0x40000000 /* disable lingering on client and server connections */ #endif /* _TYPES_BACKEND_H */ diff --git a/include/types/global.h b/include/types/global.h index b259954..340b583 100644 --- a/include/types/global.h +++ b/include/types/global.h @@ -55,6 +55,7 @@ struct global { int rlimit_memmax; /* default ulimit-d in megs value : 0=unset */ int mode; int last_checks; + int spread_checks; char *chroot; char *pidfile; int logfac1, logfac2; @@ -73,6 +74,7 @@ extern int listeners; extern char trash[BUFSIZE]; extern const int zero; extern const int one; +extern const struct linger nolinger; extern int stopping; /* non zero means stopping in progress */ #endif /* _TYPES_GLOBAL_H */ diff --git a/src/backend.c b/src/backend.c index da8b8ac..91c0b34 100644 --- a/src/backend.c +++ b/src/backend.c @@ -408,6 +408,9 @@ int connect_server(struct session *s) if (s->be->options & PR_O_TCP_SRV_KA) setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (char *) &one, sizeof(one)); + if (s->be->options & PR_O_TCP_NOLING) + setsockopt(fd, SOL_SOCKET, SO_LINGER, (struct linger *) &nolinger, sizeof(struct linger)); + /* allow specific binding : * - server-specific at first * - proxy-specific next diff --git a/src/cfgparse.c b/src/cfgparse.c index 8c9c2c6..4cb7111 100644 --- a/src/cfgparse.c +++ b/src/cfgparse.c @@ -92,6 +92,7 @@ static const struct { { "redispatch", PR_O_REDISP, PR_CAP_BE, 0 }, { "keepalive", PR_O_KEEPALIVE, PR_CAP_NONE, 0 }, { "httpclose", PR_O_HTTP_CLOSE, PR_CAP_FE | PR_CAP_BE, 0 }, + { "nolinger", PR_O_TCP_NOLING, PR_CAP_FE | PR_CAP_BE, 0 }, { "logasap", PR_O_LOGASAP, PR_CAP_FE, 0 }, { "abortonclose", PR_O_ABRT_CLOSE, PR_CAP_BE, 0 }, { "checkcache", PR_O_CHK_CACHE, PR_CAP_BE, 0 }, @@ -450,7 +451,21 @@ int cfg_parse_global(const char *file, int linenum, char **args) Alert("parsing [%s:%d] : too many syslog servers\n", file, linenum); return -1; } - + } + else if (!strcmp(args[0], "spread-checks")) { /* random time between checks (0-50) */ + if (global.spread_checks != 0) { + Alert("parsing [%s:%d]: spread-checks already specified. Continuing.\n", file, linenum); + return 0; + } + if (*(args[1]) == 0) { + Alert("parsing [%s:%d]: '%s' expects an integer argument (0..50).\n", file, linenum, args[0]); + return -1; + } + global.spread_checks = atol(args[1]); + if (global.spread_checks < 0 || global.spread_checks > 50) { + Alert("parsing [%s:%d]: 'spread-checks' needs a positive value in range 0..50.\n", file, linenum); + return -1; + } } else { Alert("parsing [%s:%d] : unknown keyword '%s' in '%s' section\n", file, linenum, args[0], "global"); @@ -2261,7 +2276,6 @@ int readcfgfile(const char *file) char *args[MAX_LINE_ARGS + 1]; int arg; int cfgerr = 0; - int nbchk, mininter; int confsect = CFG_NONE; struct proxy *curproxy = NULL; @@ -2708,56 +2722,6 @@ int readcfgfile(const char *file) newsrv = newsrv->next; } - /* now we'll start this proxy's health checks if any */ - /* 1- count the checkers to run simultaneously */ - nbchk = 0; - mininter = 0; - newsrv = curproxy->srv; - while (newsrv != NULL) { - if (newsrv->state & SRV_CHECKED) { - if (!mininter || mininter > newsrv->inter) - mininter = newsrv->inter; - nbchk++; - } - newsrv = newsrv->next; - } - - /* 2- start them as far as possible from each others while respecting - * their own intervals. For this, we will start them after their own - * interval added to the min interval divided by the number of servers, - * weighted by the server's position in the list. - */ - if (nbchk > 0) { - struct task *t; - int srvpos; - - newsrv = curproxy->srv; - srvpos = 0; - while (newsrv != NULL) { - /* should this server be checked ? */ - if (newsrv->state & SRV_CHECKED) { - if ((t = pool_alloc2(pool2_task)) == NULL) { - Alert("parsing [%s:%d] : out of memory.\n", file, linenum); - return -1; - } - - t->wq = NULL; - t->qlist.p = NULL; - t->state = TASK_IDLE; - t->process = process_chk; - t->context = newsrv; - - /* check this every ms */ - tv_ms_add(&t->expire, &now, - newsrv->inter + mininter * srvpos / nbchk); - task_queue(t); - //task_wakeup(&rq, t); - srvpos++; - } - newsrv = newsrv->next; - } - } - curproxy = curproxy->next; } if (cfgerr > 0) { diff --git a/src/checks.c b/src/checks.c index 2c581ef..cd6bd30 100644 --- a/src/checks.c +++ b/src/checks.c @@ -13,7 +13,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -210,7 +212,6 @@ static int event_srv_chk_w(int fd) static int event_srv_chk_r(int fd) { __label__ out_wakeup; - char reply[64]; int len, result; struct task *t = fdtab[fd].owner; struct server *s = t->context; @@ -230,13 +231,13 @@ static int event_srv_chk_r(int fd) } #ifndef MSG_NOSIGNAL - len = recv(fd, reply, sizeof(reply), 0); + len = recv(fd, trash, sizeof(trash), 0); #else /* Warning! Linux returns EAGAIN on SO_ERROR if data are still available * but the connection was closed on the remote end. Fortunately, recv still * works correctly and we don't need to do the getsockopt() on linux. */ - len = recv(fd, reply, sizeof(reply), MSG_NOSIGNAL); + len = recv(fd, trash, sizeof(trash), MSG_NOSIGNAL); #endif if (unlikely(len < 0 && errno == EAGAIN)) { /* we want some polling to happen first */ @@ -245,17 +246,17 @@ static int event_srv_chk_r(int fd) } if ((s->proxy->options & PR_O_HTTP_CHK) && (len >= sizeof("HTTP/1.0 000")) && - (memcmp(reply, "HTTP/1.", 7) == 0) && (reply[9] == '2' || reply[9] == '3')) { + (memcmp(trash, "HTTP/1.", 7) == 0) && (trash[9] == '2' || trash[9] == '3')) { /* HTTP/1.X 2xx or 3xx */ result = 1; } else if ((s->proxy->options & PR_O_SSL3_CHK) && (len >= 5) && - (reply[0] == 0x15 || reply[0] == 0x16)) { + (trash[0] == 0x15 || trash[0] == 0x16)) { /* SSLv3 alert or handshake */ result = 1; } else if ((s->proxy->options & PR_O_SMTP_CHK) && (len >= 3) && - (reply[0] == '2')) /* 2xx (should be 250) */ { + (trash[0] == '2')) /* 2xx (should be 250) */ { result = 1; } @@ -282,6 +283,7 @@ void process_chk(struct task *t, struct timeval *next) struct server *s = t->context; struct sockaddr_in sa; int fd; + int rv; //fprintf(stderr, "process_chk: task=%p\n", t); @@ -489,8 +491,14 @@ void process_chk(struct task *t, struct timeval *next) } s->curfd = -1; /* no check running anymore */ fd_delete(fd); - while (tv_isle(&t->expire, &now)) - tv_ms_add(&t->expire, &t->expire, s->inter); + + rv = 0; + if (global.spread_checks > 0) { + rv = s->inter * global.spread_checks / 100; + rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0))); + //fprintf(stderr, "process_chk(%p): (%d+/-%d%%) random=%d\n", s, s->inter, global.spread_checks, rv); + } + tv_ms_add(&t->expire, &now, s->inter + rv); goto new_chk; } else if (s->result < 0 || tv_isle(&t->expire, &now)) { @@ -504,8 +512,14 @@ void process_chk(struct task *t, struct timeval *next) set_server_down(s); s->curfd = -1; fd_delete(fd); - while (tv_isle(&t->expire, &now)) - tv_ms_add(&t->expire, &t->expire, s->inter); + + rv = 0; + if (global.spread_checks > 0) { + rv = s->inter * global.spread_checks / 100; + rv -= (int) (2 * rv * (rand() / (RAND_MAX + 1.0))); + //fprintf(stderr, "process_chk(%p): (%d+/-%d%%) random=%d\n", s, s->inter, global.spread_checks, rv); + } + tv_ms_add(&t->expire, &now, s->inter + rv); goto new_chk; } /* if result is 0 and there's no timeout, we have to wait again */ @@ -518,6 +532,73 @@ void process_chk(struct task *t, struct timeval *next) return; } +/* + * Start health-check. + * Returns 0 if OK, -1 if error, and prints the error in this case. + */ +int start_checks() { + + struct proxy *px; + struct server *s; + struct task *t; + int nbchk=0, mininter=0, srvpos=0; + + /* 1- count the checkers to run simultaneously. + * We also determine the minimum interval among all of those which + * have an interval larger than SRV_CHK_INTER_THRES. This interval + * will be used to spread their start-up date. Those which have + * a shorter interval will start independantly and will not dictate + * too short an interval for all others. + */ + for (px = proxy; px; px = px->next) { + for (s = px->srv; s; s = s->next) { + if (!(s->state & SRV_CHECKED)) + continue; + + if ((s->inter >= SRV_CHK_INTER_THRES) && + (!mininter || mininter > s->inter)) + mininter = s->inter; + + nbchk++; + } + } + + if (!nbchk) + return 0; + + srand((unsigned)time(NULL)); + + /* + * 2- start them as far as possible from each others. For this, we will + * start them after their interval set to the min interval divided by + * the number of servers, weighted by the server's position in the list. + */ + for (px = proxy; px; px = px->next) { + for (s = px->srv; s; s = s->next) { + if (!(s->state & SRV_CHECKED)) + continue; + + if ((t = pool_alloc2(pool2_task)) == NULL) { + Alert("Starting [%s:%s] check: out of memory.\n", px->id, s->id); + return -1; + } + + t->wq = NULL; + t->qlist.p = NULL; + t->state = TASK_IDLE; + t->process = process_chk; + t->context = s; + + /* check this every ms */ + tv_ms_add(&t->expire, &now, + ((mininter && mininter >= s->inter) ? mininter : s->inter) * srvpos / nbchk); + task_queue(t); + + srvpos++; + } + } + return 0; +} /* * Local variables: diff --git a/src/client.c b/src/client.c index 87495a4..1f58154 100644 --- a/src/client.c +++ b/src/client.c @@ -165,6 +165,9 @@ int event_accept(int fd) { if (p->options & PR_O_TCP_CLI_KA) setsockopt(cfd, SOL_SOCKET, SO_KEEPALIVE, (char *) &one, sizeof(one)); + if (p->options & PR_O_TCP_NOLING) + setsockopt(cfd, SOL_SOCKET, SO_LINGER, (struct linger *) &nolinger, sizeof(struct linger)); + t->wq = NULL; t->qlist.p = NULL; t->state = TASK_IDLE; diff --git a/src/haproxy.c b/src/haproxy.c index fe83630..7b7a691 100644 --- a/src/haproxy.c +++ b/src/haproxy.c @@ -81,6 +81,7 @@ #include #include #include +#include #include #include #include @@ -135,6 +136,7 @@ char trash[BUFSIZE]; const int zero = 0; const int one = 1; +const struct linger nolinger = { .l_onoff = 1, .l_linger = 0 }; /* * Syslog facilities and levels. Conforming to RFC3164. @@ -505,6 +507,7 @@ void init(int argc, char **argv) Alert("Error reading configuration file : %s\n", cfg_cfgfile); exit(1); } + if (have_appsession) appsession_init(); @@ -513,6 +516,9 @@ void init(int argc, char **argv) exit(0); } + if (start_checks() < 0) + exit(1); + if (cfg_maxconn > 0) global.maxconn = cfg_maxconn; @@ -983,6 +989,7 @@ int main(int argc, char **argv) if (pidfile != NULL) fclose(pidfile); free(global.pidfile); + global.pidfile = NULL; if (proc == global.nbproc) exit(0); /* parent must leave */ diff --git a/src/proto_http.c b/src/proto_http.c index 3d83a11..8a1083f 100644 --- a/src/proto_http.c +++ b/src/proto_http.c @@ -2866,7 +2866,7 @@ int process_srv(struct session *t) * Cache-Control or Expires header fields." */ if (likely(txn->meth != HTTP_METH_POST) && - unlikely(t->be->options & PR_O_CHK_CACHE)) + (t->be->options & (PR_O_CHK_CACHE|PR_O_COOK_NOC))) txn->flags |= TX_CACHEABLE | TX_CACHE_COOK; break; default: @@ -3003,8 +3003,15 @@ int process_srv(struct session *t) */ manage_server_side_cookies(t, rep); + /* - * 5: add server cookie in the response if needed + * 5: check for cache-control or pragma headers. + */ + check_response_for_cacheability(t, rep); + + + /* + * 6: add server cookie in the response if needed */ if ((t->srv) && !(t->flags & SN_DIRECT) && (t->be->options & PR_O_COOK_INS) && (!(t->be->options & PR_O_COOK_POST) || (txn->meth == HTTP_METH_POST))) { @@ -3029,7 +3036,10 @@ int process_srv(struct session *t) * Some caches understand the correct form: 'no-cache="set-cookie"', but * others don't (eg: apache <= 1.3.26). So we use 'private' instead. */ - if (t->be->options & PR_O_COOK_NOC) { + if ((t->be->options & PR_O_COOK_NOC) && (txn->flags & TX_CACHEABLE)) { + + txn->flags &= ~TX_CACHEABLE & ~TX_CACHE_COOK; + if (unlikely(http_header_add_tail2(rep, &txn->rsp, &txn->hdr_idx, "Cache-control: private", 22)) < 0) goto return_bad_resp; @@ -3038,12 +3048,6 @@ int process_srv(struct session *t) /* - * 6: check for cache-control or pragma headers. - */ - check_response_for_cacheability(t, rep); - - - /* * 7: check if result will be cacheable with a cookie. * We'll block the response if security checks have caught * nasty things such as a cacheable cookie. diff --git a/src/proxy.c b/src/proxy.c index 7d4b2ec..bd33c84 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -113,6 +114,9 @@ int start_proxies(int verbose) Alert("cannot do so_reuseaddr for proxy %s. Continuing.\n", curproxy->id); } + + if (curproxy->options & PR_O_TCP_NOLING) + setsockopt(fd, SOL_SOCKET, SO_LINGER, (struct linger *) &nolinger, sizeof(struct linger)); #ifdef SO_REUSEPORT /* OpenBSD supports this. As it's present in old libc versions of Linux,