Merge pull request #219 from pi-hole/release/3.0

Pi-hole FTL v3.0

.gitignore

@@ -1,6 +1,7 @@
 # Generated binary files
 /obj/
 pihole-FTL
+socket-test
 
 # Versioning files
 version*
@@ -10,4 +11,10 @@ version*
 /cmake-build-debug/
 
 # IDE files
-.idea/
+.idea/
+
+# Test-generated files
+/pihole.log
+/pihole-FTL.conf
+/pihole-FTL.db
+/pihole-FTL.log

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "test/libs/bats"]
+	path = test/libs/bats
+	url = https://github.com/sstephenson/bats
+[submodule "test/libs/bats-support"]
+	path = test/libs/bats-support
+	url = https://github.com/ztombol/bats-support

FTL.h

@@ -40,6 +40,12 @@
 #include "sqlite3.h"
 // tolower()
 #include <ctype.h>
+// Unix socket
+#include <sys/un.h>
+// Interfaces
+#include <ifaddrs.h>
+#include <net/if.h>
+
 
 #include "routines.h"
 
@@ -54,10 +60,6 @@
 
 #define SOCKETBUFFERLEN 1024
 
-// Maximum time from now until we will parse logs that are in the past [seconds]
-// Default: 86400 (24 hours)
-#define MAXLOGAGE 86400
-
 // How often do we garbage collect (to ensure we only have data fitting to the MAXLOGAGE defined above)? [seconds]
 // Default: 3600 (once per hour)
 #define GCinterval 3600
@@ -66,18 +68,21 @@
 // Default -60 (one minute before a full hour)
 #define GCdelay (-60)
 
+// How many client connection do we accept at once?
+#define MAXCONNS 20
+
 // Static structs
 typedef struct {
 	const char* conf;
 	const char* log;
 	const char* pid;
 	const char* port;
 	char* db;
+	const char* socketfile;
 } FTLFileNamesStruct;
 
 typedef struct {
 	const char* log;
-	const char* log1;
 	const char* gravity;
 	const char* whitelist;
 	const char* blacklist;
@@ -106,22 +111,24 @@ typedef struct {
 	int overTime;
 	int IPv4;
 	int IPv6;
-	int PTR;
-	int SRV;
 	int wildcarddomains;
 	int forwardedqueries;
+	int reply_NODATA;
+	int reply_NXDOMAIN;
+	int reply_CNAME;
+	int reply_IP;
 } countersStruct;
 
 typedef struct {
 	bool socket_listenlocal;
-	bool include_yesterday;
-	bool rolling_24h;
 	bool query_display;
 	bool analyze_AAAA;
 	int maxDBdays;
 	bool resolveIPv6;
 	bool resolveIPv4;
 	int DBinterval;
+	int port;
+	int maxlogage;
 } ConfigStruct;
 
 // Dynamic structs
@@ -137,6 +144,11 @@ typedef struct {
 	int forwardID;
 	bool valid;
 	bool db;
+	// the ID is a (signed) int in dnsmasq, so no need for a long int here
+	int id;
+	bool complete;
+	unsigned char reply;
+	int generation;
 } queriesDataStruct;
 
 typedef struct {
@@ -159,6 +171,7 @@ typedef struct {
 	int blockedcount;
 	char *domain;
 	bool wildcard;
+	unsigned char dnssec;
 } domainsDataStruct;
 
 typedef struct {
@@ -186,8 +199,15 @@ typedef struct {
 	int querytypedata;
 } memoryStruct;
 
+// Prepare timers, used mainly for debugging purposes
+#define NUMTIMERS 2
+enum { DATABASE_WRITE_TIMER, EXIT_TIMER };
+
 enum { QUERIES, FORWARDED, CLIENTS, DOMAINS, OVERTIME, WILDCARD };
-enum { SOCKET };
+enum { DNSSEC_UNSPECIFIED, DNSSEC_SECURE, DNSSEC_INSECURE, DNSSEC_BOGUS, DNSSEC_ABANDONED, DNSSEC_UNKNOWN };
+
+// Used to check memory integrity in various structs
+#define MAGICBYTE 0x57
 
 logFileNamesStruct files;
 FTLFileNamesStruct FTLfiles;
@@ -231,3 +251,17 @@ long int lastdbindex;
 bool travis;
 bool DBdeleteoldqueries;
 bool rereadgravity;
+long int lastDBimportedtimestamp;
+bool ipv4telnet, ipv6telnet;
+bool istelnet[MAXCONNS];
+
+// Use out own memory handling functions that will detect possible errors
+// and report accordingly in the log. This will make debugging FTL crashs
+// caused by insufficient memory or by code bugs (not properly dealing
+// with NULL pointers) much easier.
+#define free(param) FTLfree(param, __FILE__,  __FUNCTION__,  __LINE__)
+#define lib_strdup() strdup()
+#undef strdup
+#define strdup(param) FTLstrdup(param, __FILE__,  __FUNCTION__,  __LINE__)
+#define calloc(p1,p2) FTLcalloc(p1,p2, __FILE__,  __FUNCTION__,  __LINE__)
+#define realloc(p1,p2) FTLrealloc(p1,p2, __FILE__,  __FUNCTION__,  __LINE__)

Makefile

@@ -8,8 +8,8 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
-DEPS = FTL.h routines.h version.h
-OBJ = main.o structs.o log.o daemon.o parser.o signals.o socket.o request.o grep.o setupVars.o args.o flush.o threads.o gc.o config.o database.o
+DEPS = FTL.h routines.h api.h version.h
+OBJ = main.o memory.o log.o daemon.o parser.o signals.o socket.o request.o grep.o setupVars.o args.o flush.o threads.o gc.o config.o database.o api.o msgpack.o
 
 # Get git commit version and date
 GIT_BRANCH := $(shell git branch | sed -n 's/^\* //p')
@@ -46,7 +46,7 @@ _DEPS = $(patsubst %,$(IDIR)/%,$(DEPS))
 
 _OBJ = $(patsubst %,$(ODIR)/%,$(OBJ))
 
-all: pihole-FTL
+all: pihole-FTL socket-test
 
 $(ODIR)/%.o: %.c $(_DEPS) | $(ODIR)
 	$(CC) -c -o $@ $< -g3 $(CCFLAGS)
@@ -60,6 +60,9 @@ $(ODIR)/sqlite3.o: sqlite3.c
 pihole-FTL: $(_OBJ) $(ODIR)/sqlite3.o
 	$(CC) -v $(CCFLAGS) -o $@ $^ $(LIBS)
 
+socket-test: socket_client.c
+	$(CC) -o $@ $< $(CCFLAGS)
+
 .PHONY: clean force install
 
 clean:

README.md

@@ -4,12 +4,14 @@
 
 ![](https://i1.wp.com/pi-hole.net/wp-content/uploads/2017/01/dominik.gif?w=128&ssl=1)
 
+The Faster-Than-Light (FTL) Engine is a lightweight, purpose-built daemon used to provide statistics needed for the Web Interface, and its API can be easily integrated into your own projects. Although it is an optional component of the Pi-hole ecosystem, it will be installed by default to provide statistics. As the name implies, FTL does its work *very quickly*!
+
+The results can be accessed via a standard Unix socket (`var/run/pihole/FTL.sock`), a `telnet`-like connection (TCP socket on port 4711) as well as indirectly via the Web API (`admin/api.php`) and Command Line (`pihole -c -j`). You can out find more details below.
+
 This project is copyright under the latest version of the EUPL.
 
 Please see `LICENSE` file for your rights under this license.
 
-[![Codacy Badge](https://api.codacy.com/project/badge/Grade/4ed93e1a06fe4600ba991ef33078973c)](https://www.codacy.com/app/Pi-hole/FTL?utm_source=github.com&utm_medium=referral&utm_content=pi-hole/FTL&utm_campaign=badger)
-
 ---
 
 ### Compatibility list
@@ -92,6 +94,7 @@ Command line arguments can be arbitrarily combined, e.g. `pihole-FTL debug test`
 - `/var/log/pihole-FTL.log` log file
 - `/var/run/pihole-FTL.pid` PID file
 - `/var/run/pihole-FTL.port` file containing port on which `FTL` is listening
+- `/var/run/pihole/FTL.sock` Unix socket
 
 ### Socket connections
 
@@ -105,14 +108,14 @@ You can create a file `/etc/pihole/pihole-FTL.conf` that will be read by `FTL` o
 Possible settings (**the option shown first is the default**):
 
 - `SOCKET_LISTENING=localonly|all` (Listen only for local socket connections or permit all connections)
-- `TIMEFRAME=rolling24h|yesterday|today` (Rolling data window, up to 48h (today + yesterday), or up to 24h (only today, as in Pi-hole `v2.x` ))
 - `QUERY_DISPLAY=yes|no` (Display all queries? Set to `no` to hide query display)
 - `AAAA_QUERY_ANALYSIS=yes|no` (Allow `FTL` to analyze AAAA queries from pihole.log?)
 - `MAXDBDAYS=365` (How long should queries be stored in the database? Setting this to `0` disables the database altogether)
 - `RESOLVE_IPV6=yes|no` (Should `FTL` try to resolve IPv6 addresses to host names?)
 - `RESOLVE_IPV4=yes|no` (Should `FTL` try to resolve IPv4 addresses to host names?)
 - `DBINTERVAL=1.0` (How often do we store queries in FTL's database [minutes]?)
 - `DBFILE=/etc/pihole/pihole-FTL.db` (Specify path and filename of FTL's SQLite long-term database. Setting this to `DBFILE=` disables the database altogether)
+- `MAXLOGAGE=24.0` (Up to how many hours of queries should be imported from the database and logs? Maximum is 744 (31 days))
 
 ### Implemented keywords (starting with `>`, subject to change):