Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Store domainlist IDs for blocked/permitted queries #1409

Merged
merged 5 commits into from
Sep 11, 2022
Merged

Store domainlist IDs for blocked/permitted queries #1409

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
15942b1
Store domainlist IDs for blocked/permitted queries
DL6ER Aug 18, 2022
da33517
Add more debugging output to getstr()
DL6ER Aug 20, 2022
ff30d4a
Log memory access error details even when no debug flag is set
DL6ER Aug 20, 2022
1ccfd9a
Ensure the API does not try to create DNS cache records for queries t…
DL6ER Aug 20, 2022
bea9630
Apply suggestions from code review
DL6ER Sep 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 6 additions & 7 deletions src/api/api.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1008,15 +1008,14 @@ void getAllQueries(const char *client_message, const int sock, const bool isteln
CNAME_domain = getCNAMEDomainString(query);
}

// Get ID of blocking regex, if applicable and permitted by privacy settings
int regex_idx = -1;
if ((query->status == QUERY_REGEX || query->status == QUERY_REGEX_CNAME) &&
config.privacylevel < PRIVACY_HIDE_DOMAINS)
// Get domainlist table ID, if applicable and permitted by privacy settings
int domainlist_id = -1;
if (config.privacylevel < PRIVACY_HIDE_DOMAINS)
{
unsigned int cacheID = findCacheID(query->domainID, query->clientID, query->type);
unsigned int cacheID = findCacheID(query->domainID, query->clientID, query->type, false);
DNSCacheData *dns_cache = getDNSCache(cacheID, true);
if(dns_cache != NULL)
regex_idx = dns_cache->black_regex_idx;
domainlist_id = dns_cache->domainlist_id;
}

// Get IP of upstream destination, if applicable
Expand Down Expand Up @@ -1065,7 +1064,7 @@ void getAllQueries(const char *client_message, const int sock, const bool isteln
reply,
delay,
CNAME_domain,
regex_idx,
domainlist_id,
upstream_name,
upstream_port,
query->ede == -1 ? "" : get_edestr(query->ede));
Expand Down
60 changes: 22 additions & 38 deletions src/database/gravity-db.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
#include "../config.h"
// logg()
#include "../log.h"
// match_regex()
#include "../regex_r.h"
// getstr()
#include "../shmem.h"
// SQLite3 prepared statement vectors
Expand Down Expand Up @@ -210,11 +208,11 @@ bool gravityDB_reopen(void)
return gravityDB_open();
}

static char* get_client_querystr(const char* table, const char* groups)
static char* get_client_querystr(const char *table, const char *column, const char *groups)
{
// Build query string with group filtering
char *querystr = NULL;
if(asprintf(&querystr, "SELECT EXISTS(SELECT domain from %s WHERE domain = ? AND group_id IN (%s));", table, groups) < 1)
if(asprintf(&querystr, "SELECT %s from %s WHERE domain = ? AND group_id IN (%s);", column, table, groups) < 1)
{
logg("get_client_querystr(%s, %s) - asprintf() error", table, groups);
return NULL;
Expand Down Expand Up @@ -858,19 +856,14 @@ bool gravityDB_prepare_client_statements(clientsData *client)
return false;

// Prepare whitelist statement
// We use SELECT EXISTS() as this is known to efficiently use the index
// We are only interested in whether the domain exists or not in the
// list but don't case about duplicates or similar. SELECT EXISTS(...)
// returns true as soon as it sees the first row from the query inside
// of EXISTS().
if(config.debug & DEBUG_DATABASE)
logg("gravityDB_open(): Preparing vw_whitelist statement for client %s", clientip);
querystr = get_client_querystr("vw_whitelist", getstr(client->groupspos));
querystr = get_client_querystr("vw_whitelist", "id", getstr(client->groupspos));
sqlite3_stmt* stmt = NULL;
int rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
if( rc != SQLITE_OK )
{
logg("gravityDB_open(\"SELECT EXISTS(... vw_whitelist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
logg("gravityDB_open(\"SELECT(... vw_whitelist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
gravityDB_close();
return false;
}
Expand All @@ -880,11 +873,11 @@ bool gravityDB_prepare_client_statements(clientsData *client)
// Prepare gravity statement
if(config.debug & DEBUG_DATABASE)
logg("gravityDB_open(): Preparing vw_gravity statement for client %s", clientip);
querystr = get_client_querystr("vw_gravity", getstr(client->groupspos));
querystr = get_client_querystr("vw_gravity", "domain", getstr(client->groupspos));
rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
if( rc != SQLITE_OK )
{
logg("gravityDB_open(\"SELECT EXISTS(... vw_gravity ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
logg("gravityDB_open(\"SELECT(... vw_gravity ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
gravityDB_close();
return false;
}
Expand All @@ -894,11 +887,11 @@ bool gravityDB_prepare_client_statements(clientsData *client)
// Prepare blacklist statement
if(config.debug & DEBUG_DATABASE)
logg("gravityDB_open(): Preparing vw_blacklist statement for client %s", clientip);
querystr = get_client_querystr("vw_blacklist", getstr(client->groupspos));
querystr = get_client_querystr("vw_blacklist", "id", getstr(client->groupspos));
rc = sqlite3_prepare_v3(gravity_db, querystr, -1, SQLITE_PREPARE_PERSISTENT, &stmt, NULL);
if( rc != SQLITE_OK )
{
logg("gravityDB_open(\"SELECT EXISTS(... vw_blacklist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
logg("gravityDB_open(\"SELECT(... vw_blacklist ...)\") - SQL error prepare: %s", sqlite3_errstr(rc));
gravityDB_close();
return false;
}
Expand Down Expand Up @@ -1146,7 +1139,7 @@ int gravityDB_count(const enum gravity_tables list)
return result;
}

static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, const char *listname)
static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, const char *listname, int *domain_id)
{
// Do not try to bind text to statement when database is not available
if(!gravityDB_opened && !gravityDB_open())
Expand Down Expand Up @@ -1180,19 +1173,21 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con
sqlite3_clear_bindings(stmt);
return LIST_NOT_AVAILABLE;
}
else if(rc != SQLITE_ROW)
else if(rc != SQLITE_ROW && rc != SQLITE_DONE)
{
// Any return code that is neither SQLITE_BUSY not SQLITE_ROW
// is a real error we should log
// Any return code that is neither SQLITE_BUSY nor SQLITE_ROW or
// SQLITE_DONE is an error we should log
logg("domain_in_list(\"%s\", %p, %s): Failed to perform step: %s",
domain, stmt, listname, sqlite3_errstr(rc));
sqlite3_reset(stmt);
sqlite3_clear_bindings(stmt);
return LIST_NOT_AVAILABLE;
}

// Get result of query "SELECT EXISTS(...)"
const int result = sqlite3_column_int(stmt, 0);
// Get result of query (if available)
const int result = (rc == SQLITE_ROW) ? sqlite3_column_int(stmt, 0) : -1;
if(domain_id != NULL)
*domain_id = result;

if(config.debug & DEBUG_DATABASE)
logg("domain_in_list(\"%s\", %p, %s): %d", domain, stmt, listname, result);
Expand All @@ -1209,8 +1204,7 @@ static enum db_result domain_in_list(const char *domain, sqlite3_stmt *stmt, con
sqlite3_clear_bindings(stmt);

// Return if domain was found in current table
// SELECT EXISTS(...) either returns 0 (false) or 1 (true).
return (result == 1) ? FOUND : NOT_FOUND;
return (rc == SQLITE_ROW) ? FOUND : NOT_FOUND;
}

void gravityDB_reload_groups(clientsData* client)
Expand Down Expand Up @@ -1268,17 +1262,7 @@ enum db_result in_whitelist(const char *domain, DNSCacheData *dns_cache, clients
// We have to check both the exact whitelist (using a prepared database statement)
// as well the compiled regex whitelist filters to check if the current domain is
// whitelisted.
enum db_result on_whitelist = domain_in_list(domain, stmt, "whitelist");

// For performance reasons, the regex evaluations is executed only if the
// exact whitelist lookup does not deliver a positive match. This is an
// optimization as the database lookup will most likely hit (a) more domains
// and (b) will be faster (given a sufficiently large number of regex
// whitelisting filters).
if(on_whitelist == NOT_FOUND)
on_whitelist = match_regex(domain, dns_cache, client->id, REGEX_WHITELIST, false) != -1;

return on_whitelist;
return domain_in_list(domain, stmt, "whitelist", &dns_cache->domainlist_id);
}

enum db_result in_gravity(const char *domain, clientsData *client)
Expand Down Expand Up @@ -1306,10 +1290,10 @@ enum db_result in_gravity(const char *domain, clientsData *client)
if(stmt == NULL)
stmt = gravity_stmt->get(gravity_stmt, client->id);

return domain_in_list(domain, stmt, "gravity");
return domain_in_list(domain, stmt, "gravity", NULL);
}

enum db_result in_blacklist(const char *domain, clientsData *client)
enum db_result in_blacklist(const char *domain, DNSCacheData *dns_cache, clientsData *client)
{
// If list statement is not ready and cannot be initialized (e.g. no
// access to the database), we return false to prevent an FTL crash
Expand All @@ -1334,7 +1318,7 @@ enum db_result in_blacklist(const char *domain, clientsData *client)
if(stmt == NULL)
stmt = blacklist_stmt->get(blacklist_stmt, client->id);

return domain_in_list(domain, stmt, "blacklist");
return domain_in_list(domain, stmt, "blacklist", &dns_cache->domainlist_id);
}

bool in_auditlist(const char *domain)
Expand All @@ -1345,7 +1329,7 @@ bool in_auditlist(const char *domain)
return false;

// We check the domain_audit table for the given domain
return domain_in_list(domain, auditlist_stmt, "auditlist") == FOUND;
return domain_in_list(domain, auditlist_stmt, "auditlist", NULL) == FOUND;
}

bool gravityDB_get_regex_client_groups(clientsData* client, const unsigned int numregex, const regexData *regex,
Expand Down
2 changes: 1 addition & 1 deletion src/database/gravity-db.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ void gravityDB_finalizeTable(void);
int gravityDB_count(const enum gravity_tables list);

enum db_result in_gravity(const char *domain, clientsData *client);
enum db_result in_blacklist(const char *domain, clientsData *client);
enum db_result in_blacklist(const char *domain, DNSCacheData *dns_cache, clientsData *client);
enum db_result in_whitelist(const char *domain, DNSCacheData *dns_cache, clientsData *client);
bool in_auditlist(const char *domain);

Expand Down
54 changes: 25 additions & 29 deletions src/database/query-table.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,9 @@ int DB_save_queries(sqlite3 *db)
sqlite3_bind_null(query_stmt, 7);
}

const int cacheID = findCacheID(query->domainID, query->clientID, query->type, false);
DNSCacheData *cache = getDNSCache(cacheID, true);

// ADDITIONAL_INFO
if(query->status == QUERY_GRAVITY_CNAME ||
query->status == QUERY_REGEX_CNAME ||
Expand All @@ -389,37 +392,29 @@ int DB_save_queries(sqlite3 *db)
sqlite3_bind_text(addinfo_stmt, 2, cname, len, SQLITE_STATIC);
if(sqlite3_step(addinfo_stmt) != SQLITE_DONE)
{
logg("Encountered error while trying to store addinfo in long-term database");
logg("Encountered error while trying to store addinfo in long-term database (CNAME)");
error = true;
break;
}
sqlite3_clear_bindings(addinfo_stmt);
sqlite3_reset(addinfo_stmt);
}
else if(query->status == QUERY_REGEX)
else if(cache != NULL && cache->domainlist_id > -1)
{
// Restore regex ID if applicable
const int cacheID = findCacheID(query->domainID, query->clientID, query->type);
DNSCacheData *cache = getDNSCache(cacheID, true);
if(cache != NULL)
{
sqlite3_bind_int(query_stmt, 8, ADDINFO_REGEX_ID);
sqlite3_bind_int(query_stmt, 9, cache->black_regex_idx);
sqlite3_bind_int(query_stmt, 8, ADDINFO_REGEX_ID);
sqlite3_bind_int(query_stmt, 9, cache->domainlist_id);

// Execute prepared addinfo statement and check if successful
sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_REGEX_ID);
sqlite3_bind_int(addinfo_stmt, 2, cache->black_regex_idx);
if(sqlite3_step(addinfo_stmt) != SQLITE_DONE)
{
logg("Encountered error while trying to store addinfo in long-term database");
error = true;
break;
}
sqlite3_clear_bindings(addinfo_stmt);
sqlite3_reset(addinfo_stmt);
// Execute prepared addinfo statement and check if successful
sqlite3_bind_int(addinfo_stmt, 1, ADDINFO_REGEX_ID);
sqlite3_bind_int(addinfo_stmt, 2, cache->domainlist_id);
if(sqlite3_step(addinfo_stmt) != SQLITE_DONE)
{
logg("Encountered error while trying to store addinfo in long-term database (domainlist_id)");
error = true;
break;
}
else
sqlite3_bind_null(query_stmt, 8);
sqlite3_clear_bindings(addinfo_stmt);
sqlite3_reset(addinfo_stmt);
}
else
{
Expand Down Expand Up @@ -933,16 +928,17 @@ void DB_read_queries(void)
query->CNAME_domainID = CNAMEdomainID;
}
}
else if(status == QUERY_REGEX)
else if(sqlite3_column_bytes(stmt, 7) != 0)
{
// QUERY_REGEX: Set ID regex which was the reason for blocking
const int cacheID = findCacheID(query->domainID, query->clientID, query->type);
// Set ID of the domainlist entry that was the reason for permitting/blocking this query
// We assume the value in this field is said ID when it is not a CNAME-related domain
// (checked above) and the value of additional_info is not NULL (0 bytes storage size)
const int cacheID = findCacheID(query->domainID, query->clientID, query->type, true);
DNSCacheData *cache = getDNSCache(cacheID, true);
// Only load if
// a) we have a chace entry
// b) the value of additional_info is not NULL (0 bytes storage size)
if(cache != NULL && sqlite3_column_bytes(stmt, 7) != 0)
cache->black_regex_idx = sqlite3_column_int(stmt, 7);
// a) we have a cache entry
if(cache != NULL)
cache->domainlist_id = sqlite3_column_int(stmt, 7);
}

// Increment status counters, we first have to add one to the count of
Expand Down
20 changes: 12 additions & 8 deletions src/datastructure.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,13 +319,13 @@ void change_clientcount(clientsData *client, int total, int blocked, int overTim
}
}

int findCacheID(int domainID, int clientID, enum query_types query_type)
int _findCacheID(const int domainID, const int clientID, const enum query_types query_type, const bool create_new, const char *func, int line, const char *file)
{
// Compare content of client against known client IP addresses
for(int cacheID = 0; cacheID < counters->dns_cache_size; cacheID++)
{
// Get cache pointer
DNSCacheData* dns_cache = getDNSCache(cacheID, true);
DNSCacheData* dns_cache = _getDNSCache(cacheID, true, line, func, file);

// Check if the returned pointer is valid before trying to access it
if(dns_cache == NULL)
Expand All @@ -339,11 +339,14 @@ int findCacheID(int domainID, int clientID, enum query_types query_type)
}
}

if(!create_new)
return -1;

// Get ID of new cache entry
const int cacheID = counters->dns_cache_size;

// Get client pointer
DNSCacheData* dns_cache = getDNSCache(cacheID, false);
DNSCacheData* dns_cache = _getDNSCache(cacheID, false, line, func, file);

if(dns_cache == NULL)
{
Expand All @@ -358,6 +361,7 @@ int findCacheID(int domainID, int clientID, enum query_types query_type)
dns_cache->clientID = clientID;
dns_cache->query_type = query_type;
dns_cache->force_reply = 0u;
dns_cache->domainlist_id = -1; // -1 = not set

// Increase counter by one
counters->dns_cache_size++;
Expand Down Expand Up @@ -562,22 +566,22 @@ static const char *query_status_str[QUERY_STATUS_MAX] = {
"SPECIAL_DOMAIN"
};

void _query_set_status(queriesData *query, const enum query_status new_status, const char *file, const int line)
void _query_set_status(queriesData *query, const enum query_status new_status, const char *func, const int line, const char *file)
{
// Debug logging
if(config.debug & DEBUG_STATUS)
{
const char *oldstr = query->status < QUERY_STATUS_MAX ? query_status_str[query->status] : "INVALID";
if(query->status == new_status)
{
logg("Query %i: status unchanged: %s (%d) in %s:%i",
query->id, oldstr, query->status, short_path(file), line);
logg("Query %i: status unchanged: %s (%d) in %s() (%s:%i)",
query->id, oldstr, query->status, func, short_path(file), line);
}
else
{
const char *newstr = new_status < QUERY_STATUS_MAX ? query_status_str[new_status] : "INVALID";
logg("Query %i: status changed: %s (%d) -> %s (%d) in %s:%i",
query->id, oldstr, query->status, newstr, new_status, short_path(file), line);
logg("Query %i: status changed: %s (%d) -> %s (%d) in %s() (%s:%i)",
query->id, oldstr, query->status, newstr, new_status, func, short_path(file), line);
}
}

Expand Down
Loading