Check-in [6e6baf524b]
Overview
Comment:Completed support for killing idle connections
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 6e6baf524be0f20295ef96020ef9acb2e22237e4
User & Date: rkeene on 2016-02-22 21:02:29
Other Links: manifest | tags
Context
2016-02-22
22:07
Added support for checking for a working C11 atomic compiler (unfortunately, GCC 4.7.x and 4.8.x are still broken due to compiler bugs) check-in: 8f0a3ed18a user: rkeene tags: trunk
21:02
Completed support for killing idle connections check-in: 6e6baf524b user: rkeene tags: trunk
18:31
Added start of socket timeout and cleanup check-in: a473650e79 user: rkeene tags: trunk
Changes

Modified Makefile from [8247ae4962] to [debc382dde].

1
2
3
4
5
6
7
8
9
CC = gcc
CFLAGS = -Wall -Werror -W -pthread -O3 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE
LDFLAGS = -pthread
LIBS = -lpthread
MIMETYPES = /etc/httpd/mime.types

PREFIX = /usr/local
prefix = $(PREFIX)
bindir = $(prefix)/bin

|







1
2
3
4
5
6
7
8
9
CC = gcc
CFLAGS = -std=gnu11 -Wall -Werror -W -pthread -O3 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE
LDFLAGS = -pthread
LIBS = -lpthread
MIMETYPES = /etc/httpd/mime.types

PREFIX = /usr/local
prefix = $(PREFIX)
bindir = $(prefix)/bin

Modified README from [ec4fc9a30e] to [4a3d6cb02a].

51
52
53
54
55
56
57








58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75

   1. Logging (CFLAGS, -DFILED_DONT_LOG=1)
	It is possible to disable ALL logging from filed.  When logging is
	completely disabled interlocks (mutexes) for the logging pointer are
	not engaged and the logging functions are not compiled at all.
	This results in a slightly smaller and faster binary









   2. Debugging (CFLAGS, -DFILED_DEBUG=1)
	This is an internal option and should only be used during development.

   3. Differing HTTP semantics (CFLAGS, -DFILED_NONBLOCK_HTTP=1)
	It is possible that some HTTP clients may not process the HTTP stream
	being delivered if they cannot write to the HTTP stream itself.  This
	has not been observed yet, but it is possible.  If these semantics are
	needed (and they should not be) then they can be enabled with this
	flag at the cost of performance

   4. MIME Types (MIMETYPES)
	For single-file convience "filed" compiles the mapping of file
	extensions (the string in the filename following its last dot ("."))
	into the executable.  This mapping comes from a file in the format of
		type1   type1_extension1 type1_extension2...
		type2   type2_extension1 type2_extension2...
		...
	However it may not be desirable to include this mapping, or it may be







>
>
>
>
>
>
>
>
|


|






|







51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83

   1. Logging (CFLAGS, -DFILED_DONT_LOG=1)
	It is possible to disable ALL logging from filed.  When logging is
	completely disabled interlocks (mutexes) for the logging pointer are
	not engaged and the logging functions are not compiled at all.
	This results in a slightly smaller and faster binary

   2. Kill idle connections (CFLAGS, -DFILED_DONT_TIMEOUT=1)
        Killing idle connections relies heavily upon C11 atomics.  This
        requires a relatively new version of GCC (4.9+) or other C compiler
        that implements this aspect of C11 and so it can be disabled at
        compile time (which is the only time it makes sense).  One day an
        alternate implementation might be present that uses a mutex instead
        of atomics at which point this documentation will be updated.

   3. Debugging (CFLAGS, -DFILED_DEBUG=1)
	This is an internal option and should only be used during development.

   4. Differing HTTP semantics (CFLAGS, -DFILED_NONBLOCK_HTTP=1)
	It is possible that some HTTP clients may not process the HTTP stream
	being delivered if they cannot write to the HTTP stream itself.  This
	has not been observed yet, but it is possible.  If these semantics are
	needed (and they should not be) then they can be enabled with this
	flag at the cost of performance

   5. MIME Types (MIMETYPES)
	For single-file convience "filed" compiles the mapping of file
	extensions (the string in the filename following its last dot ("."))
	into the executable.  This mapping comes from a file in the format of
		type1   type1_extension1 type1_extension2...
		type2   type2_extension1 type2_extension2...
		...
	However it may not be desirable to include this mapping, or it may be

Modified filed.c from [932c017b84] to [59492d048f].

38
39
40
41
42
43
44





45
46
47
48
49
50
51
#include <getopt.h>
#include <stdarg.h>
#include <fcntl.h>
#include <stdio.h>
#include <errno.h>
#include <time.h>
#include <pwd.h>






/* Compile time constants */
#define FILED_VERSION "1.13"
#define FILED_SENDFILE_MAX 16777215
#define FILED_MAX_FAILURE_COUNT 30
#define FILED_DEFAULT_TYPE "application/octet-stream"
#define FILED_PATH_BUFFER_SIZE 1010







>
>
>
>
>







38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
#include <getopt.h>
#include <stdarg.h>
#include <fcntl.h>
#include <stdio.h>
#include <errno.h>
#include <time.h>
#include <pwd.h>

#ifndef FILED_DONT_TIMEOUT
#include <stdatomic.h>
#include <stdbool.h>
#endif

/* Compile time constants */
#define FILED_VERSION "1.13"
#define FILED_SENDFILE_MAX 16777215
#define FILED_MAX_FAILURE_COUNT 30
#define FILED_DEFAULT_TYPE "application/octet-stream"
#define FILED_PATH_BUFFER_SIZE 1010
583
584
585
586
587
588
589
590




















































































591






592

593
594

595


























596
597
598
599
600
601
602
603






















604
605
606
607
608

609
610
611
612
613
614
615
	pthread_create(&thread_id, NULL, filed_logging_thread, args);

	filed_log_msg("START");

	return(0);
}
#endif





















































































static void *filed_sockettimeout_thread(void *arg) {






	while (1) {

		usleep(300000);
	}




























	return(NULL);

	/* NOTREACHED */
	arg = arg;
}

static int filed_sockettimeout_thread_init(void) {
	pthread_t thread_id;























	pthread_create(&thread_id, NULL, filed_sockettimeout_thread, NULL);

	return(0);
}


/* Format time per RFC2616 */
static char *filed_format_time(char *buffer, size_t buffer_len, const time_t timeinfo) {
	struct tm timeinfo_tm, *timeinfo_tm_p;

	timeinfo_tm_p = gmtime_r(&timeinfo, &timeinfo_tm);
	if (timeinfo_tm_p == NULL) {








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

>
>
>
>
>
>

>
|
|
>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>


|





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>





>







588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
	pthread_create(&thread_id, NULL, filed_logging_thread, args);

	filed_log_msg("START");

	return(0);
}
#endif

#ifdef FILED_DONT_TIMEOUT
#define filed_sockettimeout_thread_init() 0
#define filed_sockettimeout_accept(x) /**/
#define filed_sockettimeout_processing_start(x) /**/
#define filed_sockettimeout_processing_end(x) /**/
#define filed_sockettimeout_close(x) /**/
#else
_Atomic time_t filed_sockettimeout_time;
struct {
	_Atomic time_t expiration_time;
	_Atomic pthread_t thread_id;
	bool valid;
}* filed_sockettimeout_sockstatus;
long filed_sockettimeout_sockstatus_length;
int filed_sockettimeout_devnull_fd;

static int filed_sockettimeout_sockfd_in_range(int sockfd) {
	if (sockfd < 3) {
		return(0);
	}

	if (sockfd > filed_sockettimeout_sockstatus_length) {
		return(0);
	}

	return(1);
}

static void filed_sockettimeout_expire(int sockfd, int length) {
	time_t now, expire;

	now = atomic_load(&filed_sockettimeout_time);

	expire = now + length;

	atomic_store(&filed_sockettimeout_sockstatus[sockfd].expiration_time, expire);

	return;
}

static void filed_sockettimeout_accept(int sockfd) {
	if (!filed_sockettimeout_sockfd_in_range(sockfd)) {
		return;
	}

	filed_sockettimeout_expire(sockfd, 60);

	atomic_store(&filed_sockettimeout_sockstatus[sockfd].thread_id, pthread_self());

	atomic_store(&filed_sockettimeout_sockstatus[sockfd].valid, true);

	return;
}

static void filed_sockettimeout_processing_start(int sockfd) {
	if (!filed_sockettimeout_sockfd_in_range(sockfd)) {
		return;
	}

	filed_sockettimeout_expire(sockfd, 86400);

	return;
}

static void filed_sockettimeout_processing_end(int sockfd) {
	if (!filed_sockettimeout_sockfd_in_range(sockfd)) {
		return;
	}

	filed_sockettimeout_expire(sockfd, 60);

	return;
}

static void filed_sockettimeout_close(int sockfd) {
	if (!filed_sockettimeout_sockfd_in_range(sockfd)) {
		return;
	}

	atomic_store(&filed_sockettimeout_sockstatus[sockfd].valid, false);

	return;
}

static void *filed_sockettimeout_thread(void *arg) {
	time_t now, expiration_time;
	pthread_t thread_id;
	long idx;
	int count;
	bool valid;

	while (1) {
		for (count = 0; count < 10; count++) {
			usleep(30000000);

			now = time(NULL);

			atomic_store(&filed_sockettimeout_time, now);
		}

		for (idx = 0; idx < filed_sockettimeout_sockstatus_length; idx++) {
			valid = atomic_load(&filed_sockettimeout_sockstatus[idx].valid);

			if (!valid) {
				continue;
			}

			expiration_time = atomic_load(&filed_sockettimeout_sockstatus[idx].expiration_time);

			thread_id = atomic_load(&filed_sockettimeout_sockstatus[idx].thread_id);

			if (expiration_time > now) {
				continue;
			}

			filed_sockettimeout_close(idx);

			dup2(filed_sockettimeout_devnull_fd, idx);

			pthread_kill(thread_id, SIGPIPE);
		}
	}

	return(NULL);

	/* NOTREACH: We don't actually take any arguments */
	arg = arg;
}

static int filed_sockettimeout_thread_init(void) {
	pthread_t thread_id;
	long maxfd, idx;

	maxfd = sysconf(_SC_OPEN_MAX);
	if (maxfd <= 0) {
		maxfd = 4096;
	}

	filed_sockettimeout_sockstatus = malloc(sizeof(*filed_sockettimeout_sockstatus) * maxfd);
	if (filed_sockettimeout_sockstatus == NULL) {
		return(-1);
	}

	for (idx = 0; idx < maxfd; idx++) {
		filed_sockettimeout_sockstatus[idx].valid = false;
	}

	filed_sockettimeout_sockstatus_length = maxfd;

	filed_sockettimeout_devnull_fd = open("/dev/null", O_RDWR);
	if (filed_sockettimeout_devnull_fd < 0) {
		return(-1);
	}

	pthread_create(&thread_id, NULL, filed_sockettimeout_thread, NULL);

	return(0);
}
#endif

/* Format time per RFC2616 */
static char *filed_format_time(char *buffer, size_t buffer_len, const time_t timeinfo) {
	struct tm timeinfo_tm, *timeinfo_tm_p;

	timeinfo_tm_p = gmtime_r(&timeinfo, &timeinfo_tm);
	if (timeinfo_tm_p == NULL) {
948
949
950
951
952
953
954


955
956
957
958
959
960
961
	/** reason must point to a globally allocated value **/
	log->reason = reason;
	log->http_code = error_number;

	filed_log_entry(log);

	/* Close connection */


	fclose(fp);

	return;
}

/* Return a redirect to index.html */
static void filed_redirect_index(FILE *fp, const char *date_current, const char *path, struct filed_log_entry *log) {







>
>







1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
	/** reason must point to a globally allocated value **/
	log->reason = reason;
	log->http_code = error_number;

	filed_log_entry(log);

	/* Close connection */
	filed_sockettimeout_close(fileno(fp));

	fclose(fp);

	return;
}

/* Return a redirect to index.html */
static void filed_redirect_index(FILE *fp, const char *date_current, const char *path, struct filed_log_entry *log) {
970
971
972
973
974
975
976


977
978
979
980
981
982
983
	/* Log redirect */
	log->reason = "redirect";
	log->http_code = http_code;

	filed_log_entry(log);

	/* Close connection */


	fclose(fp);

	return;

	/* Currently unused: path */
	path = path;
}







>
>







1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
	/* Log redirect */
	log->reason = "redirect";
	log->http_code = http_code;

	filed_log_entry(log);

	/* Close connection */
	filed_sockettimeout_close(fileno(fp));

	fclose(fp);

	return;

	/* Currently unused: path */
	path = path;
}
1007
1008
1009
1010
1011
1012
1013


1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033


1034
1035
1036
1037
1038
1039
1040

	/* Determine current time */
	date_current = filed_format_time(date_current_b, sizeof(date_current_b), time(NULL));

	/* Open socket as ANSI I/O for ease of use */
	fp = fdopen(fd, "w+b");
	if (fp == NULL) {


		close(fd);

		log->buffer[0] = '\0';
		log->http_code = -1;
		log->reason = "fdopen_failed";

		filed_log_entry(log);

		return(FILED_CONNECTION_CLOSE);
	}

	request = filed_get_http_request(fp, request, options);

	if (request == NULL) {
		log->buffer[0] = '\0';

		filed_error_page(fp, date_current, 500, FILED_REQUEST_METHOD_GET, "format", log);

		return(FILED_CONNECTION_CLOSE);
	}



	path = request->path;
	strcpy(log->buffer, path);
	log->method = request->method;

	/* If the requested path is a directory, redirect to index page */
	if (request->type == FILED_REQUEST_TYPE_DIRECTORY) {







>
>




















>
>







1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194

	/* Determine current time */
	date_current = filed_format_time(date_current_b, sizeof(date_current_b), time(NULL));

	/* Open socket as ANSI I/O for ease of use */
	fp = fdopen(fd, "w+b");
	if (fp == NULL) {
		filed_sockettimeout_close(fd);

		close(fd);

		log->buffer[0] = '\0';
		log->http_code = -1;
		log->reason = "fdopen_failed";

		filed_log_entry(log);

		return(FILED_CONNECTION_CLOSE);
	}

	request = filed_get_http_request(fp, request, options);

	if (request == NULL) {
		log->buffer[0] = '\0';

		filed_error_page(fp, date_current, 500, FILED_REQUEST_METHOD_GET, "format", log);

		return(FILED_CONNECTION_CLOSE);
	}

	filed_sockettimeout_processing_start(fd);

	path = request->path;
	strcpy(log->buffer, path);
	log->method = request->method;

	/* If the requested path is a directory, redirect to index page */
	if (request->type == FILED_REQUEST_TYPE_DIRECTORY) {
1181
1182
1183
1184
1185
1186
1187


1188
1189
1190
1191


1192
1193
1194
1195
1196
1197
1198
	log->sent_length = sendfile_sent;

	filed_log_entry(log);

	close(fileinfo->fd);

	if (request->headers.connection != FILED_CONNECTION_KEEP_ALIVE) {


		fclose(fp);

		return(FILED_CONNECTION_CLOSE);
	}



	return(FILED_CONNECTION_KEEP_ALIVE);
}

/* Handle incoming connections */
static void *filed_worker_thread(void *arg_v) {
	struct filed_worker_thread_args *arg;







>
>




>
>







1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
	log->sent_length = sendfile_sent;

	filed_log_entry(log);

	close(fileinfo->fd);

	if (request->headers.connection != FILED_CONNECTION_KEEP_ALIVE) {
		filed_sockettimeout_close(fd);

		fclose(fp);

		return(FILED_CONNECTION_CLOSE);
	}

	filed_sockettimeout_processing_end(fd);

	return(FILED_CONNECTION_KEEP_ALIVE);
}

/* Handle incoming connections */
static void *filed_worker_thread(void *arg_v) {
	struct filed_worker_thread_args *arg;
1245
1246
1247
1248
1249
1250
1251


1252
1253
1254
1255
1256
1257
1258

			failure_count++;

			filed_log_free(log);

			continue;
		}



		/* Fill in log structure */
		if (filed_log_ip((struct sockaddr *) &addr, log->ip, sizeof(log->ip)) == NULL) {
			log->ip[0] = '\0';
			log->port = 0;
		} else {
			log->port = addr.sin6_port;







>
>







1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418

			failure_count++;

			filed_log_free(log);

			continue;
		}

		filed_sockettimeout_accept(fd);

		/* Fill in log structure */
		if (filed_log_ip((struct sockaddr *) &addr, log->ip, sizeof(log->ip)) == NULL) {
			log->ip[0] = '\0';
			log->port = 0;
		} else {
			log->port = addr.sin6_port;