Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -1,7 +1,7 @@ CC = gcc -CFLAGS = -I. -std=gnu11 -Wall -W -pthread -O3 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE $(FILED_EXTRA_CFLAGS) +CFLAGS = -I. -Wall -W -pthread -O3 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE $(FILED_EXTRA_CFLAGS) LDFLAGS = -pthread $(FILED_EXTRA_LDFLAGS) LIBS = -lpthread $(FILED_EXTRA_LIBS) MIMETYPES = /etc/httpd/mime.types PREFIX = /usr/local Index: README ================================================================== --- README +++ README @@ -53,43 +53,35 @@ It is possible to disable ALL logging from filed. When logging is completely disabled interlocks (mutexes) for the logging pointer are not engaged and the logging functions are not compiled at all. This results in a slightly smaller and faster binary - 2. Kill idle connections (CFLAGS, -DFILED_DONT_TIMEOUT=1) - Killing idle connections relies heavily upon C11 atomics. This - requires a relatively new version of GCC (4.9+) or other C compiler - that implements this aspect of C11 and so it can be disabled at - compile time (which is the only time it makes sense). One day an - alternate implementation might be present that uses a mutex instead - of atomics at which point this documentation will be updated. - - 3. Debugging (CFLAGS, -DFILED_DEBUG=1) + 2. Debugging (CFLAGS, -DFILED_DEBUG=1) This is an internal option and should only be used during development. - 4. Differing HTTP semantics (CFLAGS, -DFILED_NONBLOCK_HTTP=1) + 3. Differing HTTP semantics (CFLAGS, -DFILED_NONBLOCK_HTTP=1) It is possible that some HTTP clients may not process the HTTP stream being delivered if they cannot write to the HTTP stream itself. This has not been observed yet, but it is possible. If these semantics are needed (and they should not be) then they can be enabled with this flag at the cost of performance. - 5. Differing chroot() semantics (CFLAGS, -DFILED_FAKE_CHROOT=1) + 4. Differing chroot() semantics (CFLAGS, -DFILED_FAKE_CHROOT=1) In some cases it is desirable to mangle paths with a path prefix rather than call chroot() at startup. This is less secure and slower and should be generally avoided -- however it may be necessary to do. In these cases the executable may be compiled with the FILED_FAKE_CHROOT C preprocessor macro defined and instead of calling chroot() all HTTP requests will have the root suffix specified as the argument to the "-r" or "--root" option prepended to them. - 6. Differing "index.html" handling (CFLAGS, -DFILED_DONT_REDIRECT_DIRECTORIES=1) + 5. Differing "index.html" handling (CFLAGS, -DFILED_DONT_REDIRECT_DIRECTORIES=1) Normally "filed" redirects users who request a directory to the index.html file in that directory so that no memory allocations are required; This option lets the server generate the new path. - 7. MIME Types (MIMETYPES) + 6. MIME Types (MIMETYPES) For single-file convenience "filed" compiles the mapping of file extensions (the string in the filename following its last dot (".")) into the executable. This mapping comes from a file in the format of type1 type1_extension1 type1_extension2... type2 type2_extension1 type2_extension2... @@ -106,20 +98,5 @@ and re-open log files, must be used. Any process may be used for writing logs but if the process does not support log rotation then it will not provide that benefit. For example, if you wish to write logs to syslogd(8) you can use logger(1), such as: # ./filed --root /www --user nobody --log '|logger -t filed' --daemon - -Troubleshooting ---------------- - 1. It won't compile, something about stdatomic.h not found or _Atomic not - a valid type. - - => This is a bug in your compiler: - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58016 - - GCC 4.7.x and 4.8.x define the macro indicating that they have C11 - support and do not define the macro that C11 requires to indicate - that C11 atomics are not available. They should define that macro. - - You can disable the features in "filed" that require C11 atomics by - defining FILED_DONT_TIMEOUT in the Makefile. Index: filed.c ================================================================== --- filed.c +++ filed.c @@ -41,35 +41,10 @@ #include #include #include #include -/* - * Determine if the C compiler supports C11 atomics - */ -#if __STDC_VERSION__ >= 201112L -# ifndef __STDC_NO_ATOMICS__ -# define FILED_FEATURE_C11_ATOMICS 1 -# endif -#endif - -/* - * If the C compiler does not support C11 atomics, disable TIMEOUT support - * since it relies upon it - */ -#ifndef FILED_FEATURE_C11_ATOMICS -# define FILED_DONT_TIMEOUT 1 -#endif - -/* - * These headers are only required for TIMEOUT support - */ -#ifndef FILED_DONT_TIMEOUT -#include -#include -#endif - /* Compile time constants */ #define FILED_VERSION "1.21" #define FILED_SENDFILE_MAX 16777215 #define FILED_MAX_FAILURE_COUNT 30 #define FILED_DEFAULT_TYPE "application/octet-stream" @@ -83,11 +58,10 @@ #define LOG_FILE "-" /* Fuzzing Test Code */ #ifdef FILED_TEST_AFL #define FILED_DONT_LOG 1 -#define FILED_DONT_TIMEOUT 1 #define pthread_create(a, x, y, z) afl_pthread_create(a, x, y, z) #define bind(x, y, z) afl_bind(x, y, z) #define socket(x, y, z) 8193 #define listen(x, y) 0 #define accept(x, y, z) afl_accept(x, y, z) @@ -206,10 +180,11 @@ char buffer[FILED_PATH_BUFFER_SIZE]; /* Items for type = TRANSFER */ int http_code; const char *reason; + time_t connecttime; time_t starttime; time_t endtime; off_t req_offset; off_t req_length; off_t sent_length; @@ -488,14 +463,15 @@ if (curr->endtime == ((time_t) -1)) { curr->endtime = now; } - fprintf(fp, "TRANSFER METHOD=%s PATH=%s SRC=%s:%i TIME.START=%llu TIME.END=%llu CODE.VALUE=%u CODE.REASON=%s REQUEST.OFFSET=%llu REQUEST.LENGTH=%llu FILE.LENGTH=%llu TRANSFER.LENGTH=%llu", + fprintf(fp, "TRANSFER METHOD=%s PATH=%s SRC=%s:%i CLIENT.TIME.CONNECT=%llu REQUEST.TIME.START=%llu REQUEST.TIME.END=%llu CODE.VALUE=%u CODE.REASON=%s REQUEST.OFFSET=%llu REQUEST.LENGTH=%llu FILE.LENGTH=%llu TRANSFER.LENGTH=%llu", method, curr->buffer, curr->ip, curr->port, + (unsigned long long) curr->connecttime, (unsigned long long) curr->starttime, (unsigned long long) curr->endtime, curr->http_code, curr->reason, (unsigned long long) curr->req_offset, (unsigned long long) curr->req_length, @@ -542,10 +518,11 @@ retval = malloc(sizeof(*retval)); if (initialize) { retval->buffer[0] = '\0'; retval->http_code = -1; + retval->connecttime = 0; retval->starttime = 0; retval->endtime = 0; retval->req_offset = 0; retval->req_length = 0; retval->sent_length = 0; @@ -635,20 +612,24 @@ #define filed_sockettimeout_thread_init() 0 #define filed_sockettimeout_init() 0 #define filed_sockettimeout_accept(x) /**/ #define filed_sockettimeout_processing_start(x) /**/ #define filed_sockettimeout_processing_end(x) /**/ -#define filed_sockettimeout_close(x) /**/ +#define filed_sockettimeout_close(x, y) /**/ #else -_Atomic time_t filed_sockettimeout_time; +time_t filed_sockettimeout_time; struct { - _Atomic time_t expiration_time; - _Atomic pthread_t thread_id; - bool valid; -}* filed_sockettimeout_sockstatus; + time_t expiration_time; + pthread_t thread_id; + enum { + filed_sockettimeout_valid, + filed_sockettimeout_invalid, + } valid; +} *filed_sockettimeout_sockstatus; long filed_sockettimeout_sockstatus_length; int filed_sockettimeout_devnull_fd; +pthread_mutex_t filed_sockettimeout_mutex = PTHREAD_MUTEX_INITIALIZER; static int filed_sockettimeout_sockfd_in_range(int sockfd) { if (sockfd < 3) { return(0); } @@ -658,62 +639,82 @@ } return(1); } -static void filed_sockettimeout_expire(int sockfd, int length) { +static void filed_sockettimeout_expire(int sockfd, int length, int lockheld) { time_t now, expire; - now = atomic_load(&filed_sockettimeout_time); + if (!lockheld) { + pthread_mutex_lock(&filed_sockettimeout_mutex); + } + + now = filed_sockettimeout_time; expire = now + length; - atomic_store(&filed_sockettimeout_sockstatus[sockfd].expiration_time, expire); + filed_sockettimeout_sockstatus[sockfd].expiration_time = expire; + + if (!lockheld) { + pthread_mutex_unlock(&filed_sockettimeout_mutex); + } return; } static void filed_sockettimeout_accept(int sockfd) { if (!filed_sockettimeout_sockfd_in_range(sockfd)) { return; } - filed_sockettimeout_expire(sockfd, 60); + pthread_mutex_lock(&filed_sockettimeout_mutex); - atomic_store(&filed_sockettimeout_sockstatus[sockfd].thread_id, pthread_self()); + filed_sockettimeout_expire(sockfd, 60, 1); - atomic_store(&filed_sockettimeout_sockstatus[sockfd].valid, true); + filed_sockettimeout_sockstatus[sockfd].thread_id = pthread_self(); + + filed_sockettimeout_sockstatus[sockfd].valid = filed_sockettimeout_valid; + + pthread_mutex_unlock(&filed_sockettimeout_mutex); return; } static void filed_sockettimeout_processing_start(int sockfd) { if (!filed_sockettimeout_sockfd_in_range(sockfd)) { return; } - filed_sockettimeout_expire(sockfd, 86400); + filed_sockettimeout_expire(sockfd, 86400, 0); return; } static void filed_sockettimeout_processing_end(int sockfd) { if (!filed_sockettimeout_sockfd_in_range(sockfd)) { return; } - filed_sockettimeout_expire(sockfd, 60); + filed_sockettimeout_expire(sockfd, 60, 0); return; } -static void filed_sockettimeout_close(int sockfd) { +static void filed_sockettimeout_close(int sockfd, int lockheld) { if (!filed_sockettimeout_sockfd_in_range(sockfd)) { return; } - atomic_store(&filed_sockettimeout_sockstatus[sockfd].valid, false); + if (!lockheld) { + pthread_mutex_lock(&filed_sockettimeout_mutex); + } + + filed_sockettimeout_sockstatus[sockfd].valid = filed_sockettimeout_invalid; + + if (!lockheld) { + pthread_mutex_unlock(&filed_sockettimeout_mutex); + } return; } static void *filed_sockettimeout_thread(void *arg) { @@ -720,44 +721,54 @@ struct timespec sleep_time; time_t now, expiration_time; pthread_t thread_id; long idx; int count; - bool valid; + int valid; + int time_interval = 30; + int check_period = 90; while (1) { - for (count = 0; count < 10; count++) { - sleep_time.tv_sec = 30; + for (count = 0; count < (check_period / time_interval); count++) { + sleep_time.tv_sec = time_interval; sleep_time.tv_nsec = 0; nanosleep(&sleep_time, NULL); + pthread_mutex_lock(&filed_sockettimeout_mutex); + now = time(NULL); - atomic_store(&filed_sockettimeout_time, now); + filed_sockettimeout_time = now; + + pthread_mutex_unlock(&filed_sockettimeout_mutex); } + + pthread_mutex_lock(&filed_sockettimeout_mutex); for (idx = 0; idx < filed_sockettimeout_sockstatus_length; idx++) { - valid = atomic_load(&filed_sockettimeout_sockstatus[idx].valid); + valid = filed_sockettimeout_sockstatus[idx].valid; - if (!valid) { + if (valid != filed_sockettimeout_valid) { continue; } - expiration_time = atomic_load(&filed_sockettimeout_sockstatus[idx].expiration_time); + expiration_time = filed_sockettimeout_sockstatus[idx].expiration_time; - thread_id = atomic_load(&filed_sockettimeout_sockstatus[idx].thread_id); + thread_id = filed_sockettimeout_sockstatus[idx].thread_id; if (expiration_time > now) { continue; } - filed_sockettimeout_close(idx); + filed_sockettimeout_close(idx, 1); dup2(filed_sockettimeout_devnull_fd, idx); pthread_kill(thread_id, SIGPIPE); } + + pthread_mutex_unlock(&filed_sockettimeout_mutex); } return(NULL); /* NOTREACH: We don't actually take any arguments */ @@ -778,20 +789,20 @@ maxfd = sysconf(_SC_OPEN_MAX); if (maxfd <= 0) { maxfd = 4096; } - filed_sockettimeout_sockstatus = malloc(sizeof(*filed_sockettimeout_sockstatus) * maxfd); + filed_sockettimeout_sockstatus_length = maxfd; + filed_sockettimeout_sockstatus = malloc(sizeof(*filed_sockettimeout_sockstatus) * filed_sockettimeout_sockstatus_length); if (filed_sockettimeout_sockstatus == NULL) { return(-1); } for (idx = 0; idx < maxfd; idx++) { - filed_sockettimeout_sockstatus[idx].valid = false; + filed_sockettimeout_sockstatus[idx].valid = filed_sockettimeout_invalid; } - filed_sockettimeout_sockstatus_length = maxfd; filed_sockettimeout_devnull_fd = open("/dev/null", O_RDWR); if (filed_sockettimeout_devnull_fd < 0) { return(-1); } @@ -1201,11 +1212,11 @@ log->http_code = error_number; filed_log_entry(log); /* Close connection */ - filed_sockettimeout_close(fileno(fp)); + filed_sockettimeout_close(fileno(fp), 0); fclose(fp); return; } @@ -1226,11 +1237,11 @@ log->http_code = http_code; filed_log_entry(log); /* Close connection */ - filed_sockettimeout_close(fileno(fp)); + filed_sockettimeout_close(fileno(fp), 0); fclose(fp); return; @@ -1260,17 +1271,20 @@ char *path; char *date_current, date_current_b[64]; int http_code; FILE *fp; + /* Indicate the connection start time */ + log->connecttime = time(NULL); + /* Determine current time */ date_current = filed_format_time(date_current_b, sizeof(date_current_b), time(NULL)); /* Open socket as ANSI I/O for ease of use */ fp = fdopen(fd, "w+b"); if (fp == NULL) { - filed_sockettimeout_close(fd); + filed_sockettimeout_close(fd, 0); close(fd); log->buffer[0] = '\0'; log->http_code = -1; @@ -1459,11 +1473,11 @@ filed_log_entry(log); close(fileinfo->fd); if (request->headers.connection != FILED_CONNECTION_KEEP_ALIVE) { - filed_sockettimeout_close(fd); + filed_sockettimeout_close(fd, 0); fclose(fp); return(FILED_CONNECTION_CLOSE); }