system/glibc
3
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update.

Browse Source 
2004-10-03  Ulrich Drepper  <drepper@redhat.com>

	* nscd/connections.c: Implement alternative loop for main thread
	which uses epoll.
	* sysdeps/unix/sysv/linux/Makefile [subdir=nscd]
	(CFLAGS-connections.c): Add -DHAVE_EPOLL.

	* nscd/Makefile (CFLAGS-initgrcache.c): Add to CFLAGS-* variables,
	don't replace old content.
This commit is contained in:
Ulrich Drepper 2004-10-03 07:33:54 +00:00
parent 1945c96f2b
commit fc03df7aa6
4 changed files with 213 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ChangeLog
View File

@ -1,5 +1,15 @@
2004-10-03 Ulrich Drepper <drepper@redhat.com>
* nscd/connections.c: Implement alternative loop for main thread
which uses epoll.
* sysdeps/unix/sysv/linux/Makefile [subdir=nscd]
(CFLAGS-connections.c): Add -DHAVE_EPOLL.
2004-10-02 Ulrich Drepper <drepper@redhat.com> 2004-10-02 Ulrich Drepper <drepper@redhat.com>
* nscd/Makefile (CFLAGS-initgrcache.c): Add to CFLAGS-* variables,
don't replace old content.
* nscd/connections.c: Rewrite handling of incoming connections. All * nscd/connections.c: Rewrite handling of incoming connections. All
are handled by one thread which then hands of the descriptors for the are handled by one thread which then hands of the descriptors for the
real work to the worker threads. real work to the worker threads.

44
nscd/Makefile
View File

@ -70,28 +70,28 @@ CFLAGS-nscd_getai.c = -fexceptions
CFLAGS-nscd_initgroups.c = -fexceptions CFLAGS-nscd_initgroups.c = -fexceptions
ifeq (yesyes,$(have-fpie)$(build-shared)) ifeq (yesyes,$(have-fpie)$(build-shared))
CFLAGS-nscd.c = -fpie CFLAGS-nscd.c += -fpie
CFLAGS-connections.c = -fpie CFLAGS-connections.c += -fpie
CFLAGS-pwdcache.c = -fpie CFLAGS-pwdcache.c += -fpie
CFLAGS-getpwnam_r.c = -fpie CFLAGS-getpwnam_r.c += -fpie
CFLAGS-getpwuid_r.c = -fpie CFLAGS-getpwuid_r.c += -fpie
CFLAGS-grpcache.c = -fpie CFLAGS-grpcache.c += -fpie
CFLAGS-getgrnam_r.c = -fpie CFLAGS-getgrnam_r.c += -fpie
CFLAGS-getgrgid_r.c = -fpie CFLAGS-getgrgid_r.c += -fpie
CFLAGS-hstcache.c = -fpie CFLAGS-hstcache.c += -fpie
CFLAGS-gethstbyad_r.c = -fpie CFLAGS-gethstbyad_r.c += -fpie
CFLAGS-gethstbynm2_r.c = -fpie CFLAGS-gethstbynm2_r.c += -fpie
CFLAGS-dbg_log.c = -fpie CFLAGS-dbg_log.c += -fpie
CFLAGS-nscd_conf.c = -fpie CFLAGS-nscd_conf.c += -fpie
CFLAGS-nscd_stat.c = -fpie CFLAGS-nscd_stat.c += -fpie
CFLAGS-cache.c = -fpie CFLAGS-cache.c += -fpie
CFLAGS-xmalloc.c = -fpie CFLAGS-xmalloc.c += -fpie
CFLAGS-xstrdup.c = -fpie CFLAGS-xstrdup.c += -fpie
CFLAGS-mem.c = -fpie CFLAGS-mem.c += -fpie
CFLAGS-nscd_setup_thread.c = -fpie CFLAGS-nscd_setup_thread.c += -fpie
CFLAGS-aicache.c = -fpie CFLAGS-aicache.c += -fpie
CFLAGS-selinux.c = -fpie CFLAGS-selinux.c += -fpie
CFLAGS-initgrcache.c = -fpie CFLAGS-initgrcache.c += -fpie
$(objpfx)nscd: $(addprefix $(objpfx),$(nscd-modules:=.o)) $(objpfx)nscd: $(addprefix $(objpfx),$(nscd-modules:=.o))
$(LINK.o) -pie -Wl,-O1 \ $(LINK.o) -pie -Wl,-O1 \

247
nscd/connections.c
View File

@ -32,6 +32,9 @@
#include <stdlib.h> #include <stdlib.h>
#include <unistd.h> #include <unistd.h>
#include <arpa/inet.h> #include <arpa/inet.h>
#ifdef HAVE_EPOLL
# include <sys/epoll.h>
#endif
#include <sys/mman.h> #include <sys/mman.h>
#include <sys/param.h> #include <sys/param.h>
#include <sys/poll.h> #include <sys/poll.h>
@ -1017,34 +1020,56 @@ handle_request: request received (Version = %d)"), req.version);
} }
static unsigned int nconns;
static void
fd_ready (int fd)
{
pthread_mutex_lock (&readylist_lock);
/* Find an empty entry in FDLIST. */
size_t inner;
for (inner = 0; inner < nconns; ++inner)
if (fdlist[inner].next == NULL)
break;
assert (inner < nconns);
fdlist[inner].fd = fd;
if (readylist == NULL)
readylist = fdlist[inner].next = &fdlist[inner];
else
{
fdlist[inner].next = readylist->next;
readylist = readylist->next = &fdlist[inner];
}
bool do_signal = true;
if (__builtin_expect (nready == 0, 0))
{
++client_queued;
do_signal = false;
}
pthread_mutex_unlock (&readylist_lock);
/* Tell one of the worker threads there is work to do. */
if (do_signal)
pthread_cond_signal (&readylist_cond);
}
/* Time a connection was accepted. */
static time_t *starttime;
static void static void
__attribute__ ((__noreturn__)) __attribute__ ((__noreturn__))
main_loop (void) main_loop_poll (void)
{ {
/* Determine how much room for descriptors we should initially
allocate. This might need to change later if we cap the number
with MAXCONN. */
const long int nfds = sysconf (_SC_OPEN_MAX);
unsigned int nconns;
#define MINCONN 32
#define MAXCONN 16384
if (nfds == -1 || nfds > MAXCONN)
nconns = MAXCONN;
else if (nfds < MINCONN)
nconns = MINCONN;
else
nconns = nfds;
struct pollfd *conns = (struct pollfd *) xmalloc (nconns struct pollfd *conns = (struct pollfd *) xmalloc (nconns
* sizeof (conns[0])); * sizeof (conns[0]));
/* We need two mirroring arrays filled with the times the connection
was accepted and a place to pass descriptors on to the worker
threads. We cannot put this in the same data structure as the
CONNS data since CONNS is passed as an array to poll(). */
time_t *starttime = (time_t *) xmalloc (nconns * sizeof (starttime[0]));
fdlist = (struct fdlist *) xcalloc (nconns, sizeof (fdlist[0]));
conns[0].fd = sock; conns[0].fd = sock;
conns[0].events = POLLRDNORM; conns[0].events = POLLRDNORM;
size_t nused = 1; size_t nused = 1;
@ -1074,17 +1099,9 @@ main_loop (void)
/* We have a new incoming connection. Accept the connection. */ /* We have a new incoming connection. Accept the connection. */
int fd = TEMP_FAILURE_RETRY (accept (sock, NULL, NULL)); int fd = TEMP_FAILURE_RETRY (accept (sock, NULL, NULL));
if (fd >= 0) /* use the descriptor if we have not reached the limit. */
if (fd >= 0 && firstfree < nconns)
{ {
/* We have a new file descriptor. Keep it around
and wait until data becomes available. */
if (firstfree == nconns)
{
// XXX Maybe extend array. For now, reject
close (fd);
goto reject_out;
}
conns[firstfree].fd = fd; conns[firstfree].fd = fd;
conns[firstfree].events = POLLRDNORM; conns[firstfree].events = POLLRDNORM;
starttime[firstfree] = now; starttime[firstfree] = now;
@ -1096,44 +1113,13 @@ main_loop (void)
while (firstfree < nused && conns[firstfree].fd != -1); while (firstfree < nused && conns[firstfree].fd != -1);
} }
reject_out:
--n; --n;
} }
for (size_t cnt = 1; cnt < nused && n > 0; ++cnt) for (size_t cnt = 1; cnt < nused && n > 0; ++cnt)
if (conns[cnt].revents != 0) if (conns[cnt].revents != 0)
{ {
pthread_mutex_lock (&readylist_lock); fd_ready (conns[cnt].fd);
/* Find an empty entry in FDLIST. */
size_t inner;
for (inner = 0; inner < nconns; ++inner)
if (fdlist[inner].next == NULL)
break;
assert (inner < nconns);
fdlist[inner].fd = conns[cnt].fd;
if (readylist == NULL)
readylist = fdlist[inner].next = &fdlist[inner];
else
{
fdlist[inner].next = readylist->next;
readylist = readylist->next = &fdlist[inner];
}
bool do_signal = true;
if (__builtin_expect (nready == 0, 0))
{
++client_queued;
do_signal = false;
}
pthread_mutex_unlock (&readylist_lock);
/* Tell one of the worker threads there is work to do. */
if (do_signal)
pthread_cond_signal (&readylist_cond);
/* Clean up the CONNS array. */ /* Clean up the CONNS array. */
conns[cnt].fd = -1; conns[cnt].fd = -1;
@ -1150,15 +1136,16 @@ main_loop (void)
/* Now find entries which have timed out. */ /* Now find entries which have timed out. */
assert (nused > 0); assert (nused > 0);
for (size_t cnt = nused - 1; cnt > 0; --cnt)
{ /* We make the timeout length depend on the number of file
/* We make the timeout length depend on the number of file descriptors currently used. */
descriptors currently used. */
#define ACCEPT_TIMEOUT \ #define ACCEPT_TIMEOUT \
(MAX_ACCEPT_TIMEOUT \ (MAX_ACCEPT_TIMEOUT \
- ((MAX_ACCEPT_TIMEOUT - MIN_ACCEPT_TIMEOUT) * nused) / nconns) - ((MAX_ACCEPT_TIMEOUT - MIN_ACCEPT_TIMEOUT) * nused) / nconns)
time_t laststart = now - ACCEPT_TIMEOUT;
time_t laststart = now - ACCEPT_TIMEOUT; for (size_t cnt = nused - 1; cnt > 0; --cnt)
{
if (conns[cnt].fd != -1 && starttime[cnt] < laststart) if (conns[cnt].fd != -1 && starttime[cnt] < laststart)
{ {
/* Remove the entry, it timed out. */ /* Remove the entry, it timed out. */
@ -1177,6 +1164,99 @@ main_loop (void)
} }
#ifdef HAVE_EPOLL
static void
main_loop_epoll (int efd)
{
struct epoll_event ev = { 0, };
int nused = 1;
size_t highest = 0;
/* Add the socket. */
ev.events = EPOLLRDNORM;
ev.data.fd = sock;
if (epoll_ctl (efd, EPOLL_CTL_ADD, sock, &ev) == -1)
/* We cannot use epoll. */
return;
while (1)
{
struct epoll_event revs[100];
# define nrevs (sizeof (revs) / sizeof (revs[0]))
int n = epoll_wait (efd, revs, nrevs, MAIN_THREAD_TIMEOUT);
time_t now = time (NULL);
for (int cnt = 0; cnt < n; ++cnt)
if (revs[cnt].data.fd == sock)
{
/* A new connection. */
int fd = TEMP_FAILURE_RETRY (accept (sock, NULL, NULL));
if (fd >= 0)
{
/* Try to add the new descriptor. */
ev.data.fd = fd;
if (fd >= nconns
|| epoll_ctl (efd, EPOLL_CTL_ADD, fd, &ev) == -1)
/* The descriptor is too large or something went
wrong. Close the descriptor. */
close (fd);
else
{
/* Remember when we accepted the connection. */
starttime[fd] = now;
if (fd > highest)
highest = fd;
++nused;
}
}
}
else
{
/* Remove the descriptor from the epoll descriptor. */
struct epoll_event ev = { 0, };
(void) epoll_ctl (efd, EPOLL_CTL_DEL, revs[cnt].data.fd, &ev);
/* Get a worked to handle the request. */
fd_ready (revs[cnt].data.fd);
/* Reset the time. */
starttime[revs[cnt].data.fd] = 0;
if (revs[cnt].data.fd == highest)
do
--highest;
while (highest > 0 && starttime[highest] == 0);
--nused;
}
/* Now look for descriptors for accepted connections which have
no reply in too long of a time. */
time_t laststart = now - ACCEPT_TIMEOUT;
for (int cnt = highest; cnt > STDERR_FILENO; --cnt)
if (cnt != sock && starttime[cnt] != 0 && starttime[cnt] < laststart)
{
/* We are waiting for this one for too long. Close it. */
struct epoll_event ev = {0, };
(void) epoll_ctl (efd, EPOLL_CTL_DEL, cnt, &ev);
(void) close (cnt);
starttime[cnt] = 0;
if (cnt == highest)
--highest;
}
else if (cnt != sock && starttime[cnt] == 0 && cnt == highest)
--highest;
}
}
#endif
/* Start all the threads we want. The initial process is thread no. 1. */ /* Start all the threads we want. The initial process is thread no. 1. */
void void
start_threads (void) start_threads (void)
@ -1216,9 +1296,36 @@ start_threads (void)
pthread_attr_destroy (&attr); pthread_attr_destroy (&attr);
/* Determine how much room for descriptors we should initially
allocate. This might need to change later if we cap the number
with MAXCONN. */
const long int nfds = sysconf (_SC_OPEN_MAX);
#define MINCONN 32
#define MAXCONN 16384
if (nfds == -1 || nfds > MAXCONN)
nconns = MAXCONN;
else if (nfds < MINCONN)
nconns = MINCONN;
else
nconns = nfds;
/* We need memory to pass descriptors on to the worker threads. */
fdlist = (struct fdlist *) xcalloc (nconns, sizeof (fdlist[0]));
/* Array to keep track when connection was accepted. */
starttime = (time_t *) xcalloc (nconns, sizeof (starttime[0]));
/* In the main thread we execute the loop which handles incoming /* In the main thread we execute the loop which handles incoming
connections. */ connections. */
main_loop (); #ifdef HAVE_EPOLL
int efd = epoll_create (100);
if (efd != -1)
{
main_loop_epoll (efd);
close (efd);
}
#endif
main_loop_poll ();
} }

4
sysdeps/unix/sysv/linux/Makefile
View File

@ -150,3 +150,7 @@ ifeq ($(subdir),rt)
CFLAGS-mq_send.c += -fexceptions CFLAGS-mq_send.c += -fexceptions
CFLAGS-mq_receive.c += -fexceptions CFLAGS-mq_receive.c += -fexceptions
endif endif
ifeq ($(subdir),nscd)
CFLAGS-connections.c += -DHAVE_EPOLL
endif