From 094f33a8c28a284b963dde241b78a6872df9d717 Mon Sep 17 00:00:00 2001 From: Samu Voutilainen Date: Fri, 24 Mar 2023 06:48:31 +0200 Subject: [PATCH 01/78] Correct declaration for query_blocked(). Fixes build on OpenSUSE Tumbleweed. Signed-off-by: Samu Voutilainen --- src/dnsmasq_interface.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 608801b8a..aae62d2a5 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -62,7 +62,7 @@ static void _query_set_reply(const unsigned int flags, const enum reply_type rep static bool _FTL_check_blocking(int queryID, int domainID, int clientID, const char* file, const int line); static unsigned long converttimeval(const struct timeval time) __attribute__((const)); static enum query_status detect_blocked_IP(const unsigned short flags, const union all_addr *addr, const queriesData *query, const domainsData *domain); -static void query_blocked(queriesData* query, domainsData* domain, clientsData* client, const unsigned char new_status); +static void query_blocked(queriesData* query, domainsData* domain, clientsData* client, const enum query_status new_status); static void FTL_forwarded(const unsigned int flags, const char *name, const union all_addr *addr, unsigned short port, const int id, const char* file, const int line); static void FTL_reply(const unsigned int flags, const char *name, const union all_addr *addr, const char* arg, const int id, const char* file, const int line); static void FTL_upstream_error(const union all_addr *addr, const unsigned int flags, const int id, const char* file, const int line); From 2f29edb8be4850beb339effea45b0fa91ab1cd92 Mon Sep 17 00:00:00 2001 From: Samu Voutilainen Date: Fri, 24 Mar 2023 06:52:21 +0200 Subject: [PATCH 02/78] Correct declaration for blockingstatus variable. Signed-off-by: Samu Voutilainen --- src/setupVars.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setupVars.h b/src/setupVars.h index 7588be649..8a1c346b1 100644 --- a/src/setupVars.h +++ b/src/setupVars.h @@ -20,6 +20,6 @@ char* find_equals(const char* s) __attribute__((pure)); void trim_whitespace(char *string); void check_blocking_status(void); -extern unsigned char blockingstatus; +extern enum blocking_status blockingstatus; #endif //SETUPVARS_H From 28ec1f04c45e0f254d701f3826787c584a4090d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Mar 2023 10:56:46 +0000 Subject: [PATCH 03/78] Bump actions/stale from 7.0.0 to 8.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 7.0.0 to 8.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v7.0.0...v8.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- .github/workflows/stale_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 9c3c9829b..e183ce17a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: issues: write steps: - - uses: actions/stale@v7.0.0 + - uses: actions/stale@v8.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/stale_pr.yml b/.github/workflows/stale_pr.yml index 7c753d7da..e9230b2c7 100644 --- a/.github/workflows/stale_pr.yml +++ b/.github/workflows/stale_pr.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v7.0.0 + - uses: actions/stale@v8.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Do not automatically mark PR/issue as stale From 5ad479b6289ff715f0ce4fa3ce16824e55d0f126 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Mar 2023 10:56:49 +0000 Subject: [PATCH 04/78] Bump actions/checkout from 3.4.0 to 3.5.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- .github/workflows/codespell.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 126208384..4815fc1db 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 - name: "Calculate required variables" id: variables @@ -90,7 +90,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 - name: "Fix ownership of repository" run: chown -R root . @@ -133,7 +133,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 - name: Get Binaries built in previous jobs uses: actions/download-artifact@v3.0.2 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 6ba65017a..a04908f3a 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index b8ecdb715..79111dddb 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: From 688e1d51127d7bd663b7e295bc36b8cc8c8f0730 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 25 Mar 2023 12:28:21 +0100 Subject: [PATCH 05/78] Add gravity parseList funtion to FTL Signed-off-by: DL6ER --- src/CMakeLists.txt | 2 + src/args.c | 18 ++++ src/gravity-tools.c | 201 ++++++++++++++++++++++++++++++++++++++++++++ src/gravity-tools.h | 13 +++ 4 files changed, 234 insertions(+) create mode 100644 src/gravity-tools.c create mode 100644 src/gravity-tools.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e71381793..160eb53b7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -130,6 +130,8 @@ set(sources FTL.h gc.c gc.h + gravity-tools.c + gravity-tools.h log.c log.h main.c diff --git a/src/args.c b/src/args.c index e5917f154..8c7b811d4 100644 --- a/src/args.c +++ b/src/args.c @@ -34,6 +34,8 @@ #include "lua/ftl_lua.h" // run_dhcp_discover() #include "dhcp-discover.h" +// gravity_parseList() +#include "gravity-tools.h" // defined in dnsmasq.c extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal); @@ -163,6 +165,22 @@ void parse_args(int argc, char* argv[]) (argc > 1 && strEndsWith(argv[1], ".db"))) exit(sqlite3_shell_main(argc, argv)); + // If the first argument is "gravity" (e.g., /usr/bin/pihole-FTL gravity), + // we offer some specialized gravity tools + if(argc > 1 && strcmp(argv[1], "gravity") == 0) + { + // pihole-FTL gravity parseList + if(argc == 6 && strcmp(argv[2], "parseList") == 0) + { + // Parse the given list and write the result to the given file + exit(gravity_parseList(argv[3], argv[4], argv[5])); + } + + printf("Incorrect usage of pihole-FTL gravity subcommand\n"); + exit(EXIT_FAILURE); + } + + // start from 1, as argv[0] is the executable name for(int i = 1; i < argc; i++) { diff --git a/src/gravity-tools.c b/src/gravity-tools.c new file mode 100644 index 000000000..ec10563bc --- /dev/null +++ b/src/gravity-tools.c @@ -0,0 +1,201 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2023 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Gravity tools collection routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "gravity-tools.h" +#include "args.h" +#include + +// Define valid domain patterns +// No need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already +// Adapted from https://stackoverflow.com/a/30007882 +// - Added "(?:...)" to form non-capturing groups (slighly faster) +#define VALID_DOMAIN_REXEX "([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" + +// supported ABP style: ||subdomain.domain.tlp^ +#define ABP_DOMAIN_REXEX "\\|\\|"VALID_DOMAIN_REXEX"\\^" + +// A list of items of common local hostnames not to report as unusable +// Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files +// but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output +#define FALSE_POSITIVES "localhost|localhost.localdomain|local|broadcasthost|localhost|ip6-localhost|ip6-loopback|lo0 localhost|ip6-localnet|ip6-mcastprefix|ip6-allnodes|ip6-allrouters|ip6-allhosts" + +// Print progress for files larger than 10 MB +// This is to avoid printing progress for small files +// which would be printed too often as affect performance +#define PRINT_PROGRESS_THRESHOLD 10*1000*1000 + +// Number of invalid domains to print before skipping the rest +#define MAX_INVALID_DOMAINS 5 + +// Regular expression flags +// - REG_EXTENDED: Use extended regular expressions +// - REG_NOSUB: Do not report subexpressions (report only success or fail in regexec) +#define REG_FLAGS REG_EXTENDED|REG_NOSUB + +int gravity_parseList(const char *infile, const char *outfile, const char *adlistID) +{ + // Open input file + FILE *fpin = fopen(infile, "r"); + if(fpin == NULL) + { + printf("WARNING: Unable to open %s for reading\n", infile); + return EXIT_FAILURE; + } + + // Open output file + FILE *fpout = fopen(outfile, "a"); + if(fpout == NULL) + { + printf("WARNING: Unable to open %s for appending\n", outfile); + fclose(fpin); + return EXIT_FAILURE; + } + + // Get size of input file + fseek(fpin, 0L, SEEK_END); + const size_t fsize = ftell(fpin); + rewind(fpin); + + // Compile regular expression to validate domains + regex_t exact_regex, abp_regex, false_positives_regex; + if(regcomp(&exact_regex, VALID_DOMAIN_REXEX, REG_FLAGS) != 0) + { + printf("WARNING: Unable to compile regular expression to validate exact domains\n"); + fclose(fpin); + fclose(fpout); + return EXIT_FAILURE; + } + if(regcomp(&abp_regex, ABP_DOMAIN_REXEX, REG_FLAGS) != 0) + { + printf("WARNING: Unable to compile regular expression to validate ABP-style domains\n"); + fclose(fpin); + fclose(fpout); + return EXIT_FAILURE; + } + if(regcomp(&false_positives_regex, FALSE_POSITIVES, REG_FLAGS) != 0) + { + printf("WARNING: Unable to compile regular expression to identify false positives\n"); + fclose(fpin); + fclose(fpout); + return EXIT_FAILURE; + } + + // Generate adlistID tail + char adlistIDtail[10]; + snprintf(adlistIDtail, sizeof(adlistIDtail), ",%s\n", adlistID); + + // Parse list file line by line + char *line = NULL; + size_t len = 0; + ssize_t read = 0; + size_t total_read = 0; + int last_progress = 0; + char *invalid_domains_list[MAX_INVALID_DOMAINS] = { NULL }; + unsigned int invalid_domains_list_len = 0; + unsigned int exact_domains = 0, abp_domains = 0, invalid_domains = 0; + while((read = getline(&line, &len, fpin)) != -1) + { + // Update total read bytes + total_read += read; + + // Skip comments + if(line[0] == '#') + continue; + + // Skip lines with only whitespace + if(strspn(line, " \t") == (size_t)read) + continue; + + // Remove trailing newline + if(line[read-1] == '\n') + line[read-1] = '\0'; + + // Skip empty lines + if(strlen(line) == 0) + continue; + + // Validate line + if(regexec(&exact_regex, line, 0, NULL, 0) == 0) + { + // Exact match + fputs(line, fpout); + fputs(adlistIDtail, fpout); + exact_domains++; + } + else if(regexec(&abp_regex, line, 0, NULL, 0) == 0) + { + // ABP-style match + fputs(line, fpout); + fputs(adlistIDtail, fpout); + abp_domains++; + } + else + { + // No match - add to list of invalid domains + if(invalid_domains_list_len < MAX_INVALID_DOMAINS) + { + // Check if we have this domain already + bool found = false; + for(unsigned int i = 0; i < invalid_domains_list_len; i++) + { + if(strcmp(invalid_domains_list[i], line) == 0) + { + found = true; + break; + } + } + + // If not found, check if this is a false + // positive and add it to the list if it is not + if(!found) + if(regexec(&false_positives_regex, line, 0, NULL, 0) != 0) + invalid_domains_list[invalid_domains_list_len++] = strdup(line); + } + invalid_domains++; + } + + // Print progress if the file is large enough + if(fsize > PRINT_PROGRESS_THRESHOLD) + { + // Calculate progress + const int progress = (int)(100.0*total_read/fsize); + // Print progress if it has changed + if(progress > last_progress) + { + printf("\rParsed %i%%", progress); + fflush(stdout); + last_progress = progress; + } + } + } + + // Print summary + printf("\n %s Parsed %u exact domains and %u ABP-style domains (ignored %u non-domain entries)\n", cli_tick(), exact_domains, abp_domains, invalid_domains); + if(invalid_domains_list_len > 0) + { + printf(" Sample of non-domain entries:\n"); + for(unsigned int i = 0; i < invalid_domains_list_len; i++) + printf(" - \"%s\"\n", invalid_domains_list[i]); + } + + // Free memory + free(line); + regfree(&exact_regex); + regfree(&abp_regex); + for(unsigned int i = 0; i < invalid_domains_list_len; i++) + if(invalid_domains_list[i] != NULL) + free(invalid_domains_list[i]); + + // Close files + fclose(fpin); + fclose(fpout); + + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/src/gravity-tools.h b/src/gravity-tools.h new file mode 100644 index 000000000..3cc021711 --- /dev/null +++ b/src/gravity-tools.h @@ -0,0 +1,13 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2023 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* Gravity tools collection prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#include "FTL.h" + +int gravity_parseList(const char *infile, const char *outfile, const char *adlistID); From 1097d75cb586bc7ea9daca42658cd973b0a334f6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 25 Mar 2023 18:36:52 +0100 Subject: [PATCH 06/78] Only match full lines in input file Signed-off-by: DL6ER --- src/gravity-tools.c | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index ec10563bc..47f260f0b 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -34,11 +34,6 @@ // Number of invalid domains to print before skipping the rest #define MAX_INVALID_DOMAINS 5 -// Regular expression flags -// - REG_EXTENDED: Use extended regular expressions -// - REG_NOSUB: Do not report subexpressions (report only success or fail in regexec) -#define REG_FLAGS REG_EXTENDED|REG_NOSUB - int gravity_parseList(const char *infile, const char *outfile, const char *adlistID) { // Open input file @@ -65,21 +60,21 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Compile regular expression to validate domains regex_t exact_regex, abp_regex, false_positives_regex; - if(regcomp(&exact_regex, VALID_DOMAIN_REXEX, REG_FLAGS) != 0) + if(regcomp(&exact_regex, VALID_DOMAIN_REXEX, REG_EXTENDED) != 0) { printf("WARNING: Unable to compile regular expression to validate exact domains\n"); fclose(fpin); fclose(fpout); return EXIT_FAILURE; } - if(regcomp(&abp_regex, ABP_DOMAIN_REXEX, REG_FLAGS) != 0) + if(regcomp(&abp_regex, ABP_DOMAIN_REXEX, REG_EXTENDED) != 0) { printf("WARNING: Unable to compile regular expression to validate ABP-style domains\n"); fclose(fpin); fclose(fpout); return EXIT_FAILURE; } - if(regcomp(&false_positives_regex, FALSE_POSITIVES, REG_FLAGS) != 0) + if(regcomp(&false_positives_regex, FALSE_POSITIVES, REG_EXTENDED | REG_NOSUB) != 0) { printf("WARNING: Unable to compile regular expression to identify false positives\n"); fclose(fpin); @@ -88,8 +83,9 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis } // Generate adlistID tail - char adlistIDtail[10]; + char adlistIDtail[16]; snprintf(adlistIDtail, sizeof(adlistIDtail), ",%s\n", adlistID); + adlistIDtail[sizeof(adlistIDtail)-1] = '\0'; // Parse list file line by line char *line = NULL; @@ -98,6 +94,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis size_t total_read = 0; int last_progress = 0; char *invalid_domains_list[MAX_INVALID_DOMAINS] = { NULL }; + const char *info = cli_info(); unsigned int invalid_domains_list_len = 0; unsigned int exact_domains = 0, abp_domains = 0, invalid_domains = 0; while((read = getline(&line, &len, fpin)) != -1) @@ -109,29 +106,33 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis if(line[0] == '#') continue; - // Skip lines with only whitespace - if(strspn(line, " \t") == (size_t)read) - continue; - // Remove trailing newline if(line[read-1] == '\n') line[read-1] = '\0'; // Skip empty lines - if(strlen(line) == 0) + const int line_len = strlen(line); + if(line_len == 0) continue; + regmatch_t match = { 0 }; // Validate line - if(regexec(&exact_regex, line, 0, NULL, 0) == 0) + if(regexec(&exact_regex, line, 1, &match, 0) == 0 && // <- Regex match + match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line { - // Exact match + // Exact match found + // Write domain to output file ... fputs(line, fpout); + // ... and append ,adlistID fputs(adlistIDtail, fpout); + // Increment counter exact_domains++; } - else if(regexec(&abp_regex, line, 0, NULL, 0) == 0) + else if(line[0] == '|' && + regexec(&abp_regex, line, 1, &match, 0) == 0 && // <- Regex match + match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line { - // ABP-style match + // ABP-style match (see comments above) fputs(line, fpout); fputs(adlistIDtail, fpout); abp_domains++; @@ -169,7 +170,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Print progress if it has changed if(progress > last_progress) { - printf("\rParsed %i%%", progress); + printf("\r %s Parsed %i%% of downloaded list", info, progress); fflush(stdout); last_progress = progress; } @@ -177,12 +178,12 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis } // Print summary - printf("\n %s Parsed %u exact domains and %u ABP-style domains (ignored %u non-domain entries)\n", cli_tick(), exact_domains, abp_domains, invalid_domains); + printf("\r %s Parsed %u exact domains and %u ABP-style domains (ignored %u non-domain entries)\n", cli_tick(), exact_domains, abp_domains, invalid_domains); if(invalid_domains_list_len > 0) { printf(" Sample of non-domain entries:\n"); for(unsigned int i = 0; i < invalid_domains_list_len; i++) - printf(" - \"%s\"\n", invalid_domains_list[i]); + printf(" - %s\n", invalid_domains_list[i]); } // Free memory From 19c4984ad237469554387c4d9590aeab1e2bbe8a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 25 Mar 2023 19:37:27 +0100 Subject: [PATCH 07/78] Enhance speed for ABP patterns (don't try to match domains when the line starts in "|") Signed-off-by: DL6ER --- src/gravity-tools.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index 47f260f0b..2e4b57d99 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -117,8 +117,9 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis regmatch_t match = { 0 }; // Validate line - if(regexec(&exact_regex, line, 1, &match, 0) == 0 && // <- Regex match - match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line + if(line[0] != '|' && // <- Not an ABP-style match + regexec(&exact_regex, line, 1, &match, 0) == 0 && // <- Regex match + match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line { // Exact match found // Write domain to output file ... @@ -128,9 +129,9 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Increment counter exact_domains++; } - else if(line[0] == '|' && - regexec(&abp_regex, line, 1, &match, 0) == 0 && // <- Regex match - match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line + else if(line[0] == '|' && // <- ABP-style match + regexec(&abp_regex, line, 1, &match, 0) == 0 && // <- Regex match + match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line { // ABP-style match (see comments above) fputs(line, fpout); @@ -155,9 +156,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // If not found, check if this is a false // positive and add it to the list if it is not - if(!found) - if(regexec(&false_positives_regex, line, 0, NULL, 0) != 0) - invalid_domains_list[invalid_domains_list_len++] = strdup(line); + if(!found && regexec(&false_positives_regex, line, 0, NULL, 0) != 0) + invalid_domains_list[invalid_domains_list_len++] = strdup(line); } invalid_domains++; } From e6ba74c65e8e632398345d059512095162db9901 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 Mar 2023 13:35:23 +0200 Subject: [PATCH 08/78] Store in the database instead of into a temporary file Signed-off-by: DL6ER --- src/gravity-tools.c | 186 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 160 insertions(+), 26 deletions(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index 2e4b57d99..b64882d93 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -11,6 +11,7 @@ #include "gravity-tools.h" #include "args.h" #include +#include "database/sqlite3.h" // Define valid domain patterns // No need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already @@ -34,25 +35,29 @@ // Number of invalid domains to print before skipping the rest #define MAX_INVALID_DOMAINS 5 -int gravity_parseList(const char *infile, const char *outfile, const char *adlistID) +int gravity_parseList(const char *infile, const char *outfile, const char *adlistIDstr) { + const char *info = cli_info(); + const char *tick = cli_tick(); + const char *cross = cli_cross(); + // Open input file FILE *fpin = fopen(infile, "r"); if(fpin == NULL) { - printf("WARNING: Unable to open %s for reading\n", infile); + printf("\r %s Unable to open %s for reading\n", cross, infile); return EXIT_FAILURE; } // Open output file - FILE *fpout = fopen(outfile, "a"); - if(fpout == NULL) + sqlite3 *db = NULL; + sqlite3_stmt *stmt = NULL; + if(sqlite3_open_v2(outfile, &db, SQLITE_OPEN_READWRITE, NULL) != SQLITE_OK) { - printf("WARNING: Unable to open %s for appending\n", outfile); + printf("\r %s Unable to open database file %s for writing\n", cross, outfile); fclose(fpin); return EXIT_FAILURE; } - // Get size of input file fseek(fpin, 0L, SEEK_END); const size_t fsize = ftell(fpin); @@ -62,30 +67,54 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis regex_t exact_regex, abp_regex, false_positives_regex; if(regcomp(&exact_regex, VALID_DOMAIN_REXEX, REG_EXTENDED) != 0) { - printf("WARNING: Unable to compile regular expression to validate exact domains\n"); + printf("\r %s Unable to compile regular expression to validate exact domains\n", cross); fclose(fpin); - fclose(fpout); + sqlite3_close(db); return EXIT_FAILURE; } if(regcomp(&abp_regex, ABP_DOMAIN_REXEX, REG_EXTENDED) != 0) { - printf("WARNING: Unable to compile regular expression to validate ABP-style domains\n"); + printf("\r %s Unable to compile regular expression to validate ABP-style domains\n", cross); fclose(fpin); - fclose(fpout); + sqlite3_close(db); return EXIT_FAILURE; } if(regcomp(&false_positives_regex, FALSE_POSITIVES, REG_EXTENDED | REG_NOSUB) != 0) { - printf("WARNING: Unable to compile regular expression to identify false positives\n"); + printf("\r %s Unable to compile regular expression to identify false positives\n", cross); fclose(fpin); - fclose(fpout); + sqlite3_close(db); return EXIT_FAILURE; } - // Generate adlistID tail - char adlistIDtail[16]; - snprintf(adlistIDtail, sizeof(adlistIDtail), ",%s\n", adlistID); - adlistIDtail[sizeof(adlistIDtail)-1] = '\0'; + // Begin transaction + if(sqlite3_exec(db, "BEGIN TRANSACTION;", NULL, NULL, NULL) != SQLITE_OK) + { + printf("\r %s Unable to begin transaction to insert domains into database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + + // Prepare SQL statement + const char *sql = "INSERT INTO gravity (domain, adlist_id) VALUES (?, ?);"; + if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) + { + printf("\r %s Unable to prepare SQL statement to insert domains into database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + + // Bind adlistID + const int adlistID = atoi(adlistIDstr); + if(sqlite3_bind_int(stmt, 2, adlistID) != SQLITE_OK) + { + printf("\r %s Unable to bind adlistID to SQL statement to insert domains into database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } // Parse list file line by line char *line = NULL; @@ -94,7 +123,6 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis size_t total_read = 0; int last_progress = 0; char *invalid_domains_list[MAX_INVALID_DOMAINS] = { NULL }; - const char *info = cli_info(); unsigned int invalid_domains_list_len = 0; unsigned int exact_domains = 0, abp_domains = 0, invalid_domains = 0; while((read = getline(&line, &len, fpin)) != -1) @@ -122,10 +150,22 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line { // Exact match found - // Write domain to output file ... - fputs(line, fpout); - // ... and append ,adlistID - fputs(adlistIDtail, fpout); + // Append domain to database using prepared statement + if(sqlite3_bind_text(stmt, 1, line, -1, SQLITE_STATIC) != SQLITE_OK) + { + printf("\r %s Unable to bind domain to SQL statement to insert domains into database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + if(sqlite3_step(stmt) != SQLITE_DONE) + { + printf("\r %s Unable to insert domain into database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + sqlite3_reset(stmt); // Increment counter exact_domains++; } @@ -134,8 +174,22 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line { // ABP-style match (see comments above) - fputs(line, fpout); - fputs(adlistIDtail, fpout); + // Append pattern to database using prepared statement + if(sqlite3_bind_text(stmt, 1, line, -1, SQLITE_STATIC) != SQLITE_OK) + { + printf("\r %s Unable to bind domain to SQL statement to insert domains into database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + if(sqlite3_step(stmt) != SQLITE_DONE) + { + printf("\r %s Unable to insert domain into database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + sqlite3_reset(stmt); abp_domains++; } else @@ -170,15 +224,94 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Print progress if it has changed if(progress > last_progress) { - printf("\r %s Parsed %i%% of downloaded list", info, progress); + printf("\r %s Processed %i%% of downloaded list", info, progress); fflush(stdout); last_progress = progress; } } } + // Finalize SQL statement + if(sqlite3_finalize(stmt) != SQLITE_OK) + { + printf("\r %s Unable to finalize SQL statement to insert domains into database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + + // Update database properties + // Are ABP patterns used? + if(abp_domains > 0) + { + sql = "INSERT OR REPLACE INTO info (property,value) VALUES ('abp_domains',1);"; + if(sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) + { + printf("\r %s Unable to update database properties in database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + } + + // Update number of domains on this list + sql = "UPDATE adlist SET number = ?, invalid_domains = ? WHERE id = ?;"; + if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) + { + printf("\r %s Unable to prepare SQL statement to update adlist properties in database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + + // Update date + if(sqlite3_bind_int(stmt, 1, exact_domains) != SQLITE_OK) + { + printf("\r %s Unable to bind number of domains to SQL statement to update adlist properties in database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + if(sqlite3_bind_int(stmt, 2, invalid_domains) != SQLITE_OK) + { + printf("\r %s Unable to bind number of invalid domains to SQL statement to update adlist properties in database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + if(sqlite3_bind_int(stmt, 3, adlistID) != SQLITE_OK) + { + printf("\r %s Unable to bind adlist ID to SQL statement to update adlist properties in database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + if(sqlite3_step(stmt) != SQLITE_DONE) + { + printf("\r %s Unable to update adlist properties in database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + if(sqlite3_finalize(stmt) != SQLITE_OK) + { + printf("\r %s Unable to finalize SQL statement to update adlist properties in database file %s\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + + // End transaction + if(sqlite3_exec(db, "END TRANSACTION", NULL, NULL, NULL) != SQLITE_OK) + { + printf("\r %s Unable to end transaction to insert domains into database file %s (database file may be corrupted)\n", cross, outfile); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } + // Print summary - printf("\r %s Parsed %u exact domains and %u ABP-style domains (ignored %u non-domain entries)\n", cli_tick(), exact_domains, abp_domains, invalid_domains); + printf("\r %s Parsed %u exact domains and %u ABP-style domains (ignored %u non-domain entries)\n", tick, exact_domains, abp_domains, invalid_domains); if(invalid_domains_list_len > 0) { printf(" Sample of non-domain entries:\n"); @@ -196,7 +329,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Close files fclose(fpin); - fclose(fpout); + sqlite3_close(db); + // Return success return EXIT_SUCCESS; } \ No newline at end of file From 1a32cb65deb73b2979b4b7adfc2f8fb287e62d67 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 Mar 2023 12:42:27 +0200 Subject: [PATCH 09/78] Add a few micro-optimizations to enhance speed of the parseList function and transform FQDN to domains. They are equivalent in this context but now they are not considered invalid any longer Signed-off-by: DL6ER --- src/gravity-tools.c | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index b64882d93..c96e8d664 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -118,6 +118,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Parse list file line by line char *line = NULL; + size_t lineno = 0; size_t len = 0; ssize_t read = 0; size_t total_read = 0; @@ -129,6 +130,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis { // Update total read bytes total_read += read; + lineno++; // Skip comments if(line[0] == '#') @@ -136,18 +138,21 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Remove trailing newline if(line[read-1] == '\n') - line[read-1] = '\0'; + line[--read] = '\0'; + + // Remove trailing dot (convert FQDN to domain) + if(line[read-1] == '.') + line[--read] = '\0'; // Skip empty lines - const int line_len = strlen(line); - if(line_len == 0) + if(line[0] == '\0') continue; regmatch_t match = { 0 }; // Validate line if(line[0] != '|' && // <- Not an ABP-style match regexec(&exact_regex, line, 1, &match, 0) == 0 && // <- Regex match - match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line + match.rm_so == 0 && match.rm_eo == read) // <- Match covers entire line { // Exact match found // Append domain to database using prepared statement @@ -171,7 +176,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis } else if(line[0] == '|' && // <- ABP-style match regexec(&abp_regex, line, 1, &match, 0) == 0 && // <- Regex match - match.rm_so == 0 && match.rm_eo == line_len) // <- Match covers entire line + match.rm_so == 0 && match.rm_eo == read) // <- Match covers entire line { // ABP-style match (see comments above) // Append pattern to database using prepared statement @@ -216,8 +221,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis invalid_domains++; } - // Print progress if the file is large enough - if(fsize > PRINT_PROGRESS_THRESHOLD) + // Print progress if the file is large enough every 100 lines + if(fsize > PRINT_PROGRESS_THRESHOLD && lineno % 100 == 1) { // Calculate progress const int progress = (int)(100.0*total_read/fsize); @@ -314,9 +319,10 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis printf("\r %s Parsed %u exact domains and %u ABP-style domains (ignored %u non-domain entries)\n", tick, exact_domains, abp_domains, invalid_domains); if(invalid_domains_list_len > 0) { - printf(" Sample of non-domain entries:\n"); + puts(" Sample of non-domain entries:"); for(unsigned int i = 0; i < invalid_domains_list_len; i++) - printf(" - %s\n", invalid_domains_list[i]); + printf(" - \"%s\"\n", invalid_domains_list[i]); + puts(""); } // Free memory From bed2e80510c42bfaecd37c2488470969ef521f22 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 Mar 2023 12:59:58 +0200 Subject: [PATCH 10/78] Allow TLD blocking using ABP style (port of core PR #5240) Signed-off-by: DL6ER --- src/gravity-tools.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index c96e8d664..f93e74641 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -17,10 +17,17 @@ // No need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already // Adapted from https://stackoverflow.com/a/30007882 // - Added "(?:...)" to form non-capturing groups (slighly faster) -#define VALID_DOMAIN_REXEX "([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" +#define TLD_PATTERN "[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" +#define SUBDOMAIN_PATTERN "([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\\.)" + +// supported exact style: subdomain.domain.tld +// SUBDOMAIN_PATTERN is mandatory for exact style, disallowing TLD blocking +#define VALID_DOMAIN_REXEX SUBDOMAIN_PATTERN"+"TLD_PATTERN // supported ABP style: ||subdomain.domain.tlp^ -#define ABP_DOMAIN_REXEX "\\|\\|"VALID_DOMAIN_REXEX"\\^" +// SUBDOMAIN_PATTERN is optional for ABP style, allowing TLD blocking: ||tld^ +// See https://github.com/pi-hole/pi-hole/pull/5240 +#define ABP_DOMAIN_REXEX "\\|\\|"SUBDOMAIN_PATTERN"*"TLD_PATTERN"\\^" // A list of items of common local hostnames not to report as unusable // Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files From 37de8662eafcfa5e83d9b1e75274b7e072afae0d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 7 Apr 2023 17:51:38 +0200 Subject: [PATCH 11/78] Simplify EDNS handling code and also interpret replies received from upstream Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 18 +++++------ src/dnsmasq_interface.c | 9 +++--- src/dnsmasq_interface.h | 4 +-- src/edns0.c | 67 ++++++++++++++++++++++++++--------------- src/edns0.h | 9 ++++-- 5 files changed, 63 insertions(+), 44 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index c10700676..50fe47a9a 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -783,7 +783,8 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server a.log.rcode = rcode; a.log.ede = ede; log_query(F_UPSTREAM | F_RCODE, "error", &a, NULL, 0); - + FTL_parse_pseudoheaders(pheader, (size_t)plen); + return resize_packet(header, n, pheader, plen); } @@ -1750,9 +1751,8 @@ void receive_query(struct listener *listen, time_t now) have_mark = get_incoming_mark(&source_addr, &dst_addr, /* istcp: */ 0, &mark); #endif //********************** Pi-hole modification **********************// - ednsData edns = { 0 }; - if (find_pseudoheader(header, (size_t)n, NULL, &pheader, NULL, NULL)) - FTL_parse_pseudoheaders(header, n, &source_addr, &edns); + if ((pheader = find_pseudoheader(header, (size_t)n, NULL, &pheader, NULL, NULL))) + FTL_parse_pseudoheaders(pheader, (size_t)n); //******************************************************************// if (extract_request(header, (size_t)n, daemon->namebuff, &type)) @@ -1763,7 +1763,7 @@ void receive_query(struct listener *listen, time_t now) log_query_mysockaddr(F_QUERY | F_FORWARD, daemon->namebuff, &source_addr, auth_dns ? "auth" : "query", type); piholeblocked = FTL_new_query(F_QUERY | F_FORWARD , daemon->namebuff, - &source_addr, auth_dns ? "auth" : "query", type, daemon->log_display_id, &edns, UDP); + &source_addr, auth_dns ? "auth" : "query", type, daemon->log_display_id, UDP); #ifdef HAVE_CONNTRACK is_single_query = 1; @@ -2298,9 +2298,9 @@ unsigned char *tcp_request(int confd, time_t now, no_cache_dnssec = 1; //********************** Pi-hole modification **********************// - ednsData edns = { 0 }; - if (find_pseudoheader(header, (size_t)size, NULL, &pheader, NULL, NULL)) - FTL_parse_pseudoheaders(header, size, &peer_addr, &edns); + unsigned char *pheader = NULL; + if ((pheader = find_pseudoheader(header, (size_t)size, NULL, &pheader, NULL, NULL))) + FTL_parse_pseudoheaders(pheader, (size_t)size); //******************************************************************// if ((gotname = extract_request(header, (unsigned int)size, daemon->namebuff, &qtype))) @@ -2320,7 +2320,7 @@ unsigned char *tcp_request(int confd, time_t now, &peer_addr, auth_dns ? "auth" : "query", qtype); piholeblocked = FTL_new_query(F_QUERY | F_FORWARD, daemon->namebuff, - &peer_addr, auth_dns ? "auth" : "query", qtype, daemon->log_display_id, &edns, TCP); + &peer_addr, auth_dns ? "auth" : "query", qtype, daemon->log_display_id, TCP); #ifdef HAVE_AUTH /* find queries for zones we're authoritative for, and answer them directly */ diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index aae62d2a5..61bb065b3 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -128,8 +128,6 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * if(!config.show_dnssec) return; - const ednsData edns = { 0 }; - // Type is overloaded with port since 2d65d55, so we have to // derive the real query type from the arg string unsigned short qtype = type; @@ -158,7 +156,7 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * arg = (char*)"dnssec-unknown"; } - _FTL_new_query(flags, name, NULL, arg, qtype, id, &edns, INTERNAL, file, line); + _FTL_new_query(flags, name, NULL, arg, qtype, id, INTERNAL, file, line); // forwarded upstream (type is used to store the upstream port) FTL_forwarded(flags, name, addr, type, id, path, line); } @@ -464,7 +462,7 @@ static bool is_pihole_domain(const char *domain) bool _FTL_new_query(const unsigned int flags, const char *name, union mysockaddr *addr, char *arg, const unsigned short qtype, const int id, - const ednsData *edns, const enum protocol proto, + const enum protocol proto, const char* file, const int line) { // Create new query in data structure @@ -596,6 +594,7 @@ bool _FTL_new_query(const unsigned int flags, const char *name, in_port_t clientPort = daemon->port; bool internal_query = false; char clientIP[ADDRSTRLEN+1] = { 0 }; + ednsData *edns = getEDNS(); if(config.edns0_ecs && edns && edns->client_set) { // Use ECS provided client @@ -3345,7 +3344,7 @@ int check_struct_sizes(void) result += check_one_struct("clientsData", sizeof(clientsData), 672, 648); result += check_one_struct("domainsData", sizeof(domainsData), 24, 20); result += check_one_struct("DNSCacheData", sizeof(DNSCacheData), 16, 16); - result += check_one_struct("ednsData", sizeof(ednsData), 72, 72); + result += check_one_struct("ednsData", sizeof(ednsData), 76, 76); result += check_one_struct("overTimeData", sizeof(overTimeData), 32, 24); result += check_one_struct("regexData", sizeof(regexData), 64, 48); result += check_one_struct("SharedMemory", sizeof(SharedMemory), 24, 12); diff --git a/src/dnsmasq_interface.h b/src/dnsmasq_interface.h index fdd5ed698..e0d11cb5b 100644 --- a/src/dnsmasq_interface.h +++ b/src/dnsmasq_interface.h @@ -23,8 +23,8 @@ void FTL_hook(unsigned int flags, const char *name, union all_addr *addr, char * #define FTL_iface(iface, addr, addrfamily) _FTL_iface(iface, addr, addrfamily, __FILE__, __LINE__) void _FTL_iface(struct irec *recviface, const union all_addr *addr, const sa_family_t addrfamily, const char* file, const int line); -#define FTL_new_query(flags, name, addr, arg, qtype, id, edns, proto) _FTL_new_query(flags, name, addr, arg, qtype, id, edns, proto, __FILE__, __LINE__) -bool _FTL_new_query(const unsigned int flags, const char *name, union mysockaddr *addr, char *arg, const unsigned short qtype, const int id, const ednsData *edns, enum protocol proto, const char* file, const int line); +#define FTL_new_query(flags, name, addr, arg, qtype, id, proto) _FTL_new_query(flags, name, addr, arg, qtype, id, proto, __FILE__, __LINE__) +bool _FTL_new_query(const unsigned int flags, const char *name, union mysockaddr *addr, char *arg, const unsigned short qtype, const int id, enum protocol proto, const char* file, const int line); #define FTL_header_analysis(header4, rcode, server, id) _FTL_header_analysis(header4, rcode, server, id, __FILE__, __LINE__) void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, const struct server *server, const int id, const char* file, const int line); diff --git a/src/edns0.c b/src/edns0.c index 35860381d..1352d0029 100644 --- a/src/edns0.c +++ b/src/edns0.c @@ -41,14 +41,26 @@ // dnsmasq option: --add-cpe-id=... #define EDNS0_CPE_ID EDNS0_OPTION_NOMCPEID -void FTL_parse_pseudoheaders(struct dns_header *header, size_t n, union mysockaddr *peer, ednsData *edns) +static ednsData edns = { 0 }; + +ednsData *getEDNS(void) { - int is_sign; - size_t plen; - unsigned char *pheader, *sizep; + if(edns.valid) + { + // Return pointer to ednsData structure and reset it for the + // next query + edns.valid = false; + return &edns; + } - // Extract additional record A.K.A. pseudoheader - if (!(pheader = find_pseudoheader(header, n, &plen, &sizep, &is_sign, NULL))) + // No valid EDNS data available + return NULL; +} + +void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) +{ + // Return early if we have no pseudoheader (a.k.a. additional records) + if (!pheader) return; // Debug logging @@ -152,6 +164,11 @@ void FTL_parse_pseudoheaders(struct dns_header *header, size_t n, union mysockad if(edns0_version != 0x00) return; + // Reset EDNS(0) data + memset(&edns, 0, sizeof(ednsData)); + edns.ede = -1; + edns.valid = true; + size_t offset; // The header is 11 bytes before the beginning of OPTION-DATA while ((offset = (p - pheader - 11u)) < rdlen && rdlen < UINT16_MAX) { @@ -225,8 +242,8 @@ void FTL_parse_pseudoheaders(struct dns_header *header, size_t n, union mysockad } // Copy data to edns struct - strncpy(edns->client, ipaddr, ADDRSTRLEN); - edns->client[ADDRSTRLEN-1] = '\0'; + strncpy(edns.client, ipaddr, ADDRSTRLEN); + edns.client[ADDRSTRLEN-1] = '\0'; // Only set the address as useful when it is not the // loopback address of the distant machine (127.0.0.0/8 or ::1) @@ -239,7 +256,7 @@ void FTL_parse_pseudoheaders(struct dns_header *header, size_t n, union mysockad } else { - edns->client_set = true; + edns.client_set = true; if(config.debug & DEBUG_EDNS0) logg("EDNS(0) CLIENT SUBNET: %s/%u - OK (IPv%u)", ipaddr, source_netmask, family == 1 ? 4 : 6); @@ -292,11 +309,11 @@ void FTL_parse_pseudoheaders(struct dns_header *header, size_t n, union mysockad else if(code == EDNS0_MAC_ADDR_BYTE && optlen == 6) { // EDNS(0) MAC address (BYTE format) - memcpy(edns->mac_byte, p, sizeof(edns->mac_byte)); - print_mac(edns->mac_text, (unsigned char*)edns->mac_byte, sizeof(edns->mac_byte)); - edns->mac_set = true; + memcpy(edns.mac_byte, p, sizeof(edns.mac_byte)); + print_mac(edns.mac_text, (unsigned char*)edns.mac_byte, sizeof(edns.mac_byte)); + edns.mac_set = true; if(config.debug & DEBUG_EDNS0) - logg("EDNS(0) MAC address (BYTE format): %s", edns->mac_text); + logg("EDNS(0) MAC address (BYTE format): %s", edns.mac_text); // Advance working pointer p += 6; @@ -304,19 +321,19 @@ void FTL_parse_pseudoheaders(struct dns_header *header, size_t n, union mysockad else if(code == EDNS0_MAC_ADDR_TEXT && optlen == 17) { // EDNS(0) MAC address (TEXT format) - memcpy(edns->mac_text, p, 17); - edns->mac_text[17] = '\0'; - if(sscanf(edns->mac_text, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", - &edns->mac_byte[0], - &edns->mac_byte[1], - &edns->mac_byte[2], - &edns->mac_byte[3], - &edns->mac_byte[4], - &edns->mac_byte[5]) == 6) + memcpy(edns.mac_text, p, 17); + edns.mac_text[17] = '\0'; + if(sscanf(edns.mac_text, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", + (unsigned char*)&edns.mac_byte[0], + (unsigned char*)&edns.mac_byte[1], + (unsigned char*)&edns.mac_byte[2], + (unsigned char*)&edns.mac_byte[3], + (unsigned char*)&edns.mac_byte[4], + (unsigned char*)&edns.mac_byte[5]) == 6) { - edns->mac_set = true; + edns.mac_set = true; if(config.debug & DEBUG_EDNS0) - logg("EDNS(0) MAC address (TEXT format): %s", edns->mac_text); + logg("EDNS(0) MAC address (TEXT format): %s", edns.mac_text); } else if(config.debug & DEBUG_EDNS0) { @@ -358,7 +375,7 @@ void FTL_parse_pseudoheaders(struct dns_header *header, size_t n, union mysockad else { if(config.debug & DEBUG_EDNS0) - logg("EDNS(0):n option %u with length %u", code, optlen); + logg("EDNS(0): option %u with length %u", code, optlen); // Not implemented, skip this record // Advance working pointer diff --git a/src/edns0.h b/src/edns0.h index 1ab2c9b42..9fd3b592e 100644 --- a/src/edns0.h +++ b/src/edns0.h @@ -11,13 +11,16 @@ #define EDNS0_HEADER typedef struct { - bool client_set; - bool mac_set; + bool client_set :1; + bool mac_set :1; + bool valid :1; char client[ADDRSTRLEN]; char mac_byte[6]; char mac_text[18]; + int ede; } ednsData; -void FTL_parse_pseudoheaders(struct dns_header *header, size_t n, union mysockaddr *peer, ednsData *edns); +ednsData *getEDNS(void); +void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen); #endif // EDNS0_HEADER From f99bda18956a7455183d07aba2bae2309932e0d8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 7 Apr 2023 18:01:28 +0200 Subject: [PATCH 12/78] Implement EDNS(0) EDE Signed-off-by: DL6ER --- src/edns0.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/edns0.c b/src/edns0.c index 1352d0029..d7797b600 100644 --- a/src/edns0.c +++ b/src/edns0.c @@ -372,6 +372,21 @@ void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) // Advance working pointer p += optlen; } + else if(code == EDNS0_OPTION_EDE && optlen == 2) + { + // EDNS(0) EDE + // https://datatracker.ietf.org/doc/rfc8914/ + + // The INFO-CODE from the EDE EDNS option is used to + // serve as an index into the "Extended DNS Error" IANA + // registry, the initial values for which are defined in + edns.ede = ntohs(((int)p[1] << 8) | p[0]); + if(config.debug & DEBUG_EDNS0) + logg("EDNS(0) EDE: %s (code %d)", edestr(edns.ede), edns.ede); + + // Advance working pointer + p += optlen; + } else { if(config.debug & DEBUG_EDNS0) From 3ac34d323bbdbe99ef605a05ca44c88efa0c8bef Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 7 Apr 2023 18:52:15 +0200 Subject: [PATCH 13/78] Use AD bit for IN/SECURE and EDE in SERVFAIL when prox for BOGUSy-dnsmasq option is used Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 2 +- src/dnsmasq_interface.c | 36 ++++++++++++++++++++++++++++++++++-- src/edns0.c | 7 +++++-- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 50fe47a9a..36bf9ee97 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -782,8 +782,8 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server union all_addr a; a.log.rcode = rcode; a.log.ede = ede; - log_query(F_UPSTREAM | F_RCODE, "error", &a, NULL, 0); FTL_parse_pseudoheaders(pheader, (size_t)plen); + log_query(F_UPSTREAM | F_RCODE, "error", &a, NULL, 0); return resize_packet(header, n, pheader, plen); } diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 61bb065b3..9c70e5055 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -76,6 +76,7 @@ static char *get_ptrname(struct in_addr *addr); static const char *check_dnsmasq_name(const char *name); // Static blocking metadata +static bool adbit = false; static const char *blockingreason = ""; static enum reply_type force_next_DNS_reply = REPLY_UNKNOWN; static int last_regex_idx = -1; @@ -2016,6 +2017,12 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al if(config.debug & DEBUG_QUERIES) logg(" EDE: %s (%d)", edestr(addr->log.ede), addr->log.ede); } + ednsData *edns = getEDNS(); + if(edns != NULL && edns->ede != EDE_UNSET) + { + query->ede = edns->ede; + log_debug(DEBUG_QUERIES, " EDE: %s (%d)", edestr(edns->ede), edns->ede); + } // Update upstream server (if applicable) if(!cached) @@ -2182,6 +2189,13 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al logg("***** Unknown upstream REPLY"); } + if(query && option_bool(OPT_DNSSEC_PROXY)) + { + // DNSSEC proxy mode is enabled. Interpret AD flag + // and set DNSSEC status accordingly + query_set_dnssec(query, adbit ? DNSSEC_SECURE : DNSSEC_INSECURE); + } + unlock_shm(); } @@ -2438,6 +2452,9 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl break; } + // Get EDNS data (if available) + ednsData *edns = getEDNS(); + // Debug logging if(config.debug & DEBUG_QUERIES) { @@ -2480,8 +2497,20 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl if(addr->log.ede != EDE_UNSET) // This function is only called if (flags & F_RCODE) logg(" EDE: %s (%d)", edestr(addr->log.ede), addr->log.ede); - } + if(edns != NULL && edns->ede != EDE_UNSET) + { + query->ede = edns->ede; + log_debug(DEBUG_QUERIES, " EDE: %s (%d)", edestr(edns->ede), edns->ede); + } + } + if(option_bool(OPT_DNSSEC_PROXY) && edns->ede >= EDE_DNSSEC_BOGUS && edns->ede <= EDE_NO_NSEC) + { + // DNSSEC proxy mode is enabled and we received a DNSSEC status + // from the upstream server. We need to update the DNSSEC status + // of the corresponding query. + query_set_dnssec(query, DNSSEC_BOGUS); + } // Set query reply query_set_reply(0, reply, addr, query, response); @@ -2561,6 +2590,9 @@ void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, // RA bit is not set and rcode is NXDOMAIN FTL_mark_externally_blocked(id, file, line); + // Check if AD bit is set in DNS header + adbit = header4 & HB4_AD; + // Store server which sent this reply if(server) { @@ -3300,7 +3332,7 @@ const char *get_edestr(const int ede) static void _query_set_dnssec(queriesData *query, const enum dnssec_status dnssec, const char *file, const int line) { // Return early if DNSSEC validation is disabled - if(!option_bool(OPT_DNSSEC_VALID)) + if(!option_bool(OPT_DNSSEC_VALID) && !option_bool(OPT_DNSSEC_PROXY)) return; if(config.debug & DEBUG_DNSSEC) diff --git a/src/edns0.c b/src/edns0.c index d7797b600..e7a20153b 100644 --- a/src/edns0.c +++ b/src/edns0.c @@ -166,7 +166,7 @@ void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) // Reset EDNS(0) data memset(&edns, 0, sizeof(ednsData)); - edns.ede = -1; + edns.ede = EDE_UNSET; edns.valid = true; size_t offset; // The header is 11 bytes before the beginning of OPTION-DATA @@ -379,7 +379,10 @@ void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) // The INFO-CODE from the EDE EDNS option is used to // serve as an index into the "Extended DNS Error" IANA - // registry, the initial values for which are defined in + // registry, the initial values for which are defined in + // this document. The value of the INFO-CODE is encoded + // as a two-octet unsigned integer in network byte + // order. edns.ede = ntohs(((int)p[1] << 8) | p[0]); if(config.debug & DEBUG_EDNS0) logg("EDNS(0) EDE: %s (code %d)", edestr(edns.ede), edns.ede); From 8a4488ceb0f6c3c1d967843c670b24cc4b02a84a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 7 Apr 2023 18:56:36 +0200 Subject: [PATCH 14/78] Ignore possible EXTRA-TEXT field in EDNS0 EDE data Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 5 +++-- src/edns0.c | 23 ++++++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 9c70e5055..06a239ffa 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2021,7 +2021,8 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al if(edns != NULL && edns->ede != EDE_UNSET) { query->ede = edns->ede; - log_debug(DEBUG_QUERIES, " EDE: %s (%d)", edestr(edns->ede), edns->ede); + if(config.debug & DEBUG_QUERIES) + logg(" EDE: %s (%d)", edestr(edns->ede), edns->ede); } // Update upstream server (if applicable) @@ -2501,7 +2502,7 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl if(edns != NULL && edns->ede != EDE_UNSET) { query->ede = edns->ede; - log_debug(DEBUG_QUERIES, " EDE: %s (%d)", edestr(edns->ede), edns->ede); + logg(" EDE: %s (%d)", edestr(edns->ede), edns->ede); } } if(option_bool(OPT_DNSSEC_PROXY) && edns->ede >= EDE_DNSSEC_BOGUS && edns->ede <= EDE_NO_NSEC) diff --git a/src/edns0.c b/src/edns0.c index e7a20153b..3bb524bed 100644 --- a/src/edns0.c +++ b/src/edns0.c @@ -372,18 +372,35 @@ void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) // Advance working pointer p += optlen; } - else if(code == EDNS0_OPTION_EDE && optlen == 2) + else if(code == EDNS0_OPTION_EDE && optlen >= 2) { // EDNS(0) EDE // https://datatracker.ietf.org/doc/rfc8914/ - + // + // 1 1 1 1 1 1 + // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + // +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + // 0: | OPTION-CODE | + // +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + // 2: | OPTION-LENGTH | + // +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + // 4: | INFO-CODE | + edns.ede = ntohs(((int)p[1] << 8) | p[0]); + // +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + // 6: / EXTRA-TEXT ... / + // +---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+---+ + // // The INFO-CODE from the EDE EDNS option is used to // serve as an index into the "Extended DNS Error" IANA // registry, the initial values for which are defined in // this document. The value of the INFO-CODE is encoded // as a two-octet unsigned integer in network byte // order. - edns.ede = ntohs(((int)p[1] << 8) | p[0]); + // + // The EXTRA-TEXT from the EDE EDNS option is ignored by + // FTL + + // Debug output if(config.debug & DEBUG_EDNS0) logg("EDNS(0) EDE: %s (code %d)", edestr(edns.ede), edns.ede); From ad85e0f3f66447f73afd6afa6271d67ff543a685 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 8 Apr 2023 12:54:54 +0200 Subject: [PATCH 15/78] Only try to interpret EDNS EDE when EDE data is available Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 06a239ffa..b2bcf8943 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2505,11 +2505,13 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl logg(" EDE: %s (%d)", edestr(edns->ede), edns->ede); } } - if(option_bool(OPT_DNSSEC_PROXY) && edns->ede >= EDE_DNSSEC_BOGUS && edns->ede <= EDE_NO_NSEC) + // Check EDNS EDE for DNSSEC status in DNSSEC proxy mode + if(option_bool(OPT_DNSSEC_PROXY) && + edns && edns->ede >= EDE_DNSSEC_BOGUS && edns->ede <= EDE_NO_NSEC) { - // DNSSEC proxy mode is enabled and we received a DNSSEC status - // from the upstream server. We need to update the DNSSEC status - // of the corresponding query. + // DNSSEC proxy mode is enabled and we received a valid DNSSEC + // status from the upstream server through ENDS EDE. We need to + // update the DNSSEC status of the corresponding query. query_set_dnssec(query, DNSSEC_BOGUS); } // Set query reply From 538c6a058ceb35f25a07381196e28d721c7615fc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 8 Apr 2023 14:30:40 +0200 Subject: [PATCH 16/78] Log if EDNS header is NULL and we are in debug mode Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 8 ++++---- src/dnsmasq_interface.c | 3 +++ src/edns0.c | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 36bf9ee97..0725e216d 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1751,8 +1751,8 @@ void receive_query(struct listener *listen, time_t now) have_mark = get_incoming_mark(&source_addr, &dst_addr, /* istcp: */ 0, &mark); #endif //********************** Pi-hole modification **********************// - if ((pheader = find_pseudoheader(header, (size_t)n, NULL, &pheader, NULL, NULL))) - FTL_parse_pseudoheaders(pheader, (size_t)n); + pheader = find_pseudoheader(header, (size_t)n, NULL, &pheader, NULL, NULL); + FTL_parse_pseudoheaders(pheader, (size_t)n); //******************************************************************// if (extract_request(header, (size_t)n, daemon->namebuff, &type)) @@ -2299,8 +2299,8 @@ unsigned char *tcp_request(int confd, time_t now, //********************** Pi-hole modification **********************// unsigned char *pheader = NULL; - if ((pheader = find_pseudoheader(header, (size_t)size, NULL, &pheader, NULL, NULL))) - FTL_parse_pseudoheaders(pheader, (size_t)size); + pheader = find_pseudoheader(header, (size_t)size, NULL, &pheader, NULL, NULL); + FTL_parse_pseudoheaders(pheader, (size_t)size); //******************************************************************// if ((gotname = extract_request(header, (unsigned int)size, daemon->namebuff, &qtype))) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index b2bcf8943..65af3796e 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2497,7 +2497,10 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl } if(addr->log.ede != EDE_UNSET) // This function is only called if (flags & F_RCODE) + { + query->ede = addr->log.ede; logg(" EDE: %s (%d)", edestr(addr->log.ede), addr->log.ede); + } if(edns != NULL && edns->ede != EDE_UNSET) { diff --git a/src/edns0.c b/src/edns0.c index 3bb524bed..644492fab 100644 --- a/src/edns0.c +++ b/src/edns0.c @@ -61,7 +61,11 @@ void FTL_parse_pseudoheaders(unsigned char *pheader, const size_t plen) { // Return early if we have no pseudoheader (a.k.a. additional records) if (!pheader) + { + if(config.debug & DEBUG_EDNS0) + logg("EDNS(0) pheader is NULL"); return; + } // Debug logging if(config.debug & DEBUG_EDNS0) From 72b4bc5041ae7e48df4c338376151784b9f15d7b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 9 Apr 2023 10:51:16 +0200 Subject: [PATCH 17/78] Analyse pseudeoheader before it might get stripped off Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 0725e216d..1f2eb2ddf 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -723,6 +723,10 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server /* Get extended RCODE. */ rcode |= sizep[2] << 4; + // Pi-hole modification: Interpret the pseudoheader before + // it might get stripped off below (added_pheader == true) + FTL_parse_pseudoheaders(pheader, (size_t)plen); + if (option_bool(OPT_CLIENT_SUBNET) && !check_source(header, plen, pheader, query_source)) { my_syslog(LOG_WARNING, _("discarding DNS reply: subnet option mismatch")); @@ -782,7 +786,6 @@ static size_t process_reply(struct dns_header *header, time_t now, struct server union all_addr a; a.log.rcode = rcode; a.log.ede = ede; - FTL_parse_pseudoheaders(pheader, (size_t)plen); log_query(F_UPSTREAM | F_RCODE, "error", &a, NULL, 0); return resize_packet(header, n, pheader, plen); From b22524e54c7149ecb9c7c50085f962b845acdc65 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 9 Apr 2023 11:32:12 +0200 Subject: [PATCH 18/78] Initial DNSSEC status should be UNSPECIFIED Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 65af3796e..8a20e29ad 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -743,10 +743,6 @@ bool _FTL_new_query(const unsigned int flags, const char *name, counters->reply[REPLY_UNKNOWN]++; // Store DNSSEC result for this domain query->dnssec = DNSSEC_UNSPECIFIED; - // Every domain is insecure in the beginning. It can get secure or bogus - // only if validation reveals this. If DNSSEC validation is not used, the - // original status (DNSSEC_UNSPECIFIED) is not changed. - query_set_dnssec(query, DNSSEC_INSECURE); query->CNAME_domainID = -1; // This query is not yet known ad forwarded or blocked query->flags.blocked = false; From f216cb0b780d06e9efcbd6648c5dbd558269b4ad Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 9 Apr 2023 12:31:10 +0200 Subject: [PATCH 19/78] Explicitly set INSECURE status for replies received either from upstream (if they are not already validated as SECURE) or from cache. This is a direct consequence from the previous commit. Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 8a20e29ad..292396cb1 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2105,6 +2105,10 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al // Save reply type and update individual reply counters query_set_reply(flags, 0, addr, query, response); + // Set DNSSEC status to INSECURE if it is still unknown + if(query->dnssec == DNSSEC_UNSPECIFIED) + query_set_dnssec(query, DNSSEC_INSECURE); + // Hereby, this query is now fully determined query->flags.complete = true; } @@ -2144,6 +2148,12 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al reply_flags = F_NEG; } } + else + { + // Set DNSSEC status to INSECURE if it is still unknown + if(query->dnssec == DNSSEC_UNSPECIFIED) + query_set_dnssec(query, DNSSEC_INSECURE); + } // Save reply type and update individual reply counters query_set_reply(reply_flags, 0, addr, query, response); From d228fbf3ad86d2402ad822c7c2fec0ee6227ea3e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 9 Apr 2023 12:36:06 +0200 Subject: [PATCH 20/78] Apply the same logic also for reverse lookups (PTR) Signed-off-by: DL6ER --- src/dnsmasq_interface.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index 292396cb1..e18b48309 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2184,6 +2184,10 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al // name = google-public-dns-a.google.com // Hence, isExactMatch is always false + // Set DNSSEC status to INSECURE if it is still unknown + if(query->dnssec == DNSSEC_UNSPECIFIED) + query_set_dnssec(query, DNSSEC_INSECURE); + // Save reply type and update individual reply counters query_set_reply(flags, 0, addr, query, response); } From d8271ad9b6fe5123d6303a43a8e27fcea145ea1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 15 Apr 2023 10:30:36 +0200 Subject: [PATCH 21/78] Trigger stale workflow on issue comments to remove stale label immediately MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index e183ce17a..40698ff8f 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,6 +4,7 @@ on: schedule: - cron: '0 8 * * *' workflow_dispatch: + issue_comment: jobs: stale: From da8d0108e5bade1a3e42316dd298469ce3ed3a2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Apr 2023 10:57:01 +0000 Subject: [PATCH 22/78] Bump actions/checkout from 3.5.0 to 3.5.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.0 to 3.5.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.0...v3.5.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 6 +++--- .github/workflows/codespell.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4815fc1db..824b16e95 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 - name: "Calculate required variables" id: variables @@ -90,7 +90,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 - name: "Fix ownership of repository" run: chown -R root . @@ -133,7 +133,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 - name: Get Binaries built in previous jobs uses: actions/download-artifact@v3.0.2 diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index a04908f3a..bf2bcb45d 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -10,7 +10,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 - name: Spell-Checking uses: codespell-project/actions-codespell@master diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 79111dddb..a9e3a1642 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -11,7 +11,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: From b5c1a845c82fda90843425918f8902b2d06191d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 16 Apr 2023 19:03:19 +0200 Subject: [PATCH 23/78] Run seperate job to trigger removal on comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 40698ff8f..8038de2bc 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,13 +2,13 @@ name: Mark stale issues on: schedule: - - cron: '0 8 * * *' + - cron: '0 8 * * *' workflow_dispatch: issue_comment: jobs: - stale: - + stale_action: + if: github.event_name != 'issue_comment' runs-on: ubuntu-latest permissions: issues: write @@ -25,3 +25,18 @@ jobs: exempt-all-issue-assignees: true operations-per-run: 300 close-issue-reason: 'not_planned' + + remove_stale: # trigger "stale" removal immediately when stale issues are commented on + if: github.event_name == 'issue_comment' + permissions: + contents: read # for actions/checkout + issues: write # to edit issues label + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3.4.0 + - name: Remove 'stale' label + run: gh issue edit ${{ github.event.issue.number }} --remove-label 'stale' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + From 71625df5e0acba88726bb4c334a06ef46414bd42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 17 Apr 2023 20:55:15 +0200 Subject: [PATCH 24/78] Use env variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 8038de2bc..b53a27aeb 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -6,6 +6,9 @@ on: workflow_dispatch: issue_comment: +env: + stale_label: stale + jobs: stale_action: if: github.event_name != 'issue_comment' @@ -20,7 +23,7 @@ jobs: days-before-stale: 30 days-before-close: 5 stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' - stale-issue-label: 'stale' + stale-issue-label: $stale_label exempt-issue-labels: 'Fixed in next release, Bug, Bug:Confirmed, Bugfix in progress, documentation needed, internal' exempt-all-issue-assignees: true operations-per-run: 300 @@ -36,7 +39,7 @@ jobs: - name: Checkout uses: actions/checkout@v3.4.0 - name: Remove 'stale' label - run: gh issue edit ${{ github.event.issue.number }} --remove-label 'stale' + run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 08a8a11a517a575d446effe4dd17210788e51f35 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Apr 2023 21:10:44 +0200 Subject: [PATCH 25/78] Update embedded Lua to 5.4.5 Signed-off-by: DL6ER --- src/lua/lapi.c | 249 ++++++++++++++++++++++----------------------- src/lua/lapi.h | 17 ++-- src/lua/lauxlib.c | 56 +++++----- src/lua/lcode.c | 137 ++++++++++++++++--------- src/lua/lcorolib.c | 4 +- src/lua/ldebug.c | 80 ++++++++------- src/lua/ldebug.h | 2 +- src/lua/ldo.c | 229 +++++++++++++++++++++++------------------ src/lua/ldo.h | 17 +++- src/lua/ldump.c | 8 +- src/lua/lfunc.c | 54 +++++----- src/lua/lfunc.h | 6 +- src/lua/lgc.c | 106 ++++++++++--------- src/lua/lgc.h | 19 ++-- src/lua/linit.c | 3 - src/lua/llex.c | 6 +- src/lua/llimits.h | 21 +++- src/lua/lmathlib.c | 10 +- src/lua/lmem.c | 68 ++++++++----- src/lua/loadlib.c | 9 +- src/lua/lobject.c | 38 ++++--- src/lua/lobject.h | 19 +++- src/lua/lopcodes.h | 2 +- src/lua/loslib.c | 36 ++++--- src/lua/lparser.c | 31 +++--- src/lua/lstate.c | 57 +++++------ src/lua/lstate.h | 25 +++-- src/lua/lstrlib.c | 2 +- src/lua/ltable.c | 8 +- src/lua/ltable.h | 1 - src/lua/ltablib.c | 2 +- src/lua/ltm.c | 38 +++---- src/lua/ltm.h | 5 +- src/lua/lua.c | 57 +++++------ src/lua/lua.h | 26 +++-- src/lua/luac.c | 10 +- src/lua/luaconf.h | 13 ++- src/lua/lualib.h | 4 - src/lua/lundump.c | 8 +- src/lua/lutf8lib.c | 27 +++-- src/lua/lvm.c | 235 ++++++++++++++++++++++++++---------------- src/lua/lvm.h | 5 + 42 files changed, 991 insertions(+), 759 deletions(-) diff --git a/src/lua/lapi.c b/src/lua/lapi.c index 5ee65792d..34e64af14 100644 --- a/src/lua/lapi.c +++ b/src/lua/lapi.c @@ -60,27 +60,28 @@ const char lua_ident[] = static TValue *index2value (lua_State *L, int idx) { CallInfo *ci = L->ci; if (idx > 0) { - StkId o = ci->func + idx; - api_check(L, idx <= L->ci->top - (ci->func + 1), "unacceptable index"); - if (o >= L->top) return &G(L)->nilvalue; + StkId o = ci->func.p + idx; + api_check(L, idx <= ci->top.p - (ci->func.p + 1), "unacceptable index"); + if (o >= L->top.p) return &G(L)->nilvalue; else return s2v(o); } else if (!ispseudo(idx)) { /* negative index */ - api_check(L, idx != 0 && -idx <= L->top - (ci->func + 1), "invalid index"); - return s2v(L->top + idx); + api_check(L, idx != 0 && -idx <= L->top.p - (ci->func.p + 1), + "invalid index"); + return s2v(L->top.p + idx); } else if (idx == LUA_REGISTRYINDEX) return &G(L)->l_registry; else { /* upvalues */ idx = LUA_REGISTRYINDEX - idx; api_check(L, idx <= MAXUPVAL + 1, "upvalue index too large"); - if (ttisCclosure(s2v(ci->func))) { /* C closure? */ - CClosure *func = clCvalue(s2v(ci->func)); + if (ttisCclosure(s2v(ci->func.p))) { /* C closure? */ + CClosure *func = clCvalue(s2v(ci->func.p)); return (idx <= func->nupvalues) ? &func->upvalue[idx-1] : &G(L)->nilvalue; } else { /* light C function or Lua function (through a hook)?) */ - api_check(L, ttislcf(s2v(ci->func)), "caller not a C function"); + api_check(L, ttislcf(s2v(ci->func.p)), "caller not a C function"); return &G(L)->nilvalue; /* no upvalues */ } } @@ -94,14 +95,15 @@ static TValue *index2value (lua_State *L, int idx) { l_sinline StkId index2stack (lua_State *L, int idx) { CallInfo *ci = L->ci; if (idx > 0) { - StkId o = ci->func + idx; - api_check(L, o < L->top, "invalid index"); + StkId o = ci->func.p + idx; + api_check(L, o < L->top.p, "invalid index"); return o; } else { /* non-positive index */ - api_check(L, idx != 0 && -idx <= L->top - (ci->func + 1), "invalid index"); + api_check(L, idx != 0 && -idx <= L->top.p - (ci->func.p + 1), + "invalid index"); api_check(L, !ispseudo(idx), "invalid index"); - return L->top + idx; + return L->top.p + idx; } } @@ -112,17 +114,12 @@ LUA_API int lua_checkstack (lua_State *L, int n) { lua_lock(L); ci = L->ci; api_check(L, n >= 0, "negative 'n'"); - if (L->stack_last - L->top > n) /* stack large enough? */ + if (L->stack_last.p - L->top.p > n) /* stack large enough? */ res = 1; /* yes; check is OK */ - else { /* no; need to grow stack */ - int inuse = cast_int(L->top - L->stack) + EXTRA_STACK; - if (inuse > LUAI_MAXSTACK - n) /* can grow without overflow? */ - res = 0; /* no */ - else /* try to grow stack */ - res = luaD_growstack(L, n, 0); - } - if (res && ci->top < L->top + n) - ci->top = L->top + n; /* adjust frame top */ + else /* need to grow stack */ + res = luaD_growstack(L, n, 0); + if (res && ci->top.p < L->top.p + n) + ci->top.p = L->top.p + n; /* adjust frame top */ lua_unlock(L); return res; } @@ -134,11 +131,11 @@ LUA_API void lua_xmove (lua_State *from, lua_State *to, int n) { lua_lock(to); api_checknelems(from, n); api_check(from, G(from) == G(to), "moving among independent states"); - api_check(from, to->ci->top - to->top >= n, "stack overflow"); - from->top -= n; + api_check(from, to->ci->top.p - to->top.p >= n, "stack overflow"); + from->top.p -= n; for (i = 0; i < n; i++) { - setobjs2s(to, to->top, from->top + i); - to->top++; /* stack already checked by previous 'api_check' */ + setobjs2s(to, to->top.p, from->top.p + i); + to->top.p++; /* stack already checked by previous 'api_check' */ } lua_unlock(to); } @@ -172,12 +169,12 @@ LUA_API lua_Number lua_version (lua_State *L) { LUA_API int lua_absindex (lua_State *L, int idx) { return (idx > 0 || ispseudo(idx)) ? idx - : cast_int(L->top - L->ci->func) + idx; + : cast_int(L->top.p - L->ci->func.p) + idx; } LUA_API int lua_gettop (lua_State *L) { - return cast_int(L->top - (L->ci->func + 1)); + return cast_int(L->top.p - (L->ci->func.p + 1)); } @@ -187,24 +184,24 @@ LUA_API void lua_settop (lua_State *L, int idx) { ptrdiff_t diff; /* difference for new top */ lua_lock(L); ci = L->ci; - func = ci->func; + func = ci->func.p; if (idx >= 0) { - api_check(L, idx <= ci->top - (func + 1), "new top too large"); - diff = ((func + 1) + idx) - L->top; + api_check(L, idx <= ci->top.p - (func + 1), "new top too large"); + diff = ((func + 1) + idx) - L->top.p; for (; diff > 0; diff--) - setnilvalue(s2v(L->top++)); /* clear new slots */ + setnilvalue(s2v(L->top.p++)); /* clear new slots */ } else { - api_check(L, -(idx+1) <= (L->top - (func + 1)), "invalid new top"); + api_check(L, -(idx+1) <= (L->top.p - (func + 1)), "invalid new top"); diff = idx + 1; /* will "subtract" index (as it is negative) */ } - api_check(L, L->tbclist < L->top, "previous pop of an unclosed slot"); - newtop = L->top + diff; - if (diff < 0 && L->tbclist >= newtop) { + api_check(L, L->tbclist.p < L->top.p, "previous pop of an unclosed slot"); + newtop = L->top.p + diff; + if (diff < 0 && L->tbclist.p >= newtop) { lua_assert(hastocloseCfunc(ci->nresults)); - luaF_close(L, newtop, CLOSEKTOP, 0); + newtop = luaF_close(L, newtop, CLOSEKTOP, 0); } - L->top = newtop; /* correct top only after closing any upvalue */ + L->top.p = newtop; /* correct top only after closing any upvalue */ lua_unlock(L); } @@ -213,10 +210,9 @@ LUA_API void lua_closeslot (lua_State *L, int idx) { StkId level; lua_lock(L); level = index2stack(L, idx); - api_check(L, hastocloseCfunc(L->ci->nresults) && L->tbclist == level, + api_check(L, hastocloseCfunc(L->ci->nresults) && L->tbclist.p == level, "no variable to close at given level"); - luaF_close(L, level, CLOSEKTOP, 0); - level = index2stack(L, idx); /* stack may be moved */ + level = luaF_close(L, level, CLOSEKTOP, 0); setnilvalue(s2v(level)); lua_unlock(L); } @@ -245,7 +241,7 @@ l_sinline void reverse (lua_State *L, StkId from, StkId to) { LUA_API void lua_rotate (lua_State *L, int idx, int n) { StkId p, t, m; lua_lock(L); - t = L->top - 1; /* end of stack segment being rotated */ + t = L->top.p - 1; /* end of stack segment being rotated */ p = index2stack(L, idx); /* start of segment */ api_check(L, (n >= 0 ? n : -n) <= (t - p + 1), "invalid 'n'"); m = (n >= 0 ? t - n : p - n - 1); /* end of prefix */ @@ -264,7 +260,7 @@ LUA_API void lua_copy (lua_State *L, int fromidx, int toidx) { api_check(L, isvalid(L, to), "invalid index"); setobj(L, to, fr); if (isupvalue(toidx)) /* function upvalue? */ - luaC_barrier(L, clCvalue(s2v(L->ci->func)), fr); + luaC_barrier(L, clCvalue(s2v(L->ci->func.p)), fr); /* LUA_REGISTRYINDEX does not need gc barrier (collector revisits it before finishing collection) */ lua_unlock(L); @@ -273,7 +269,7 @@ LUA_API void lua_copy (lua_State *L, int fromidx, int toidx) { LUA_API void lua_pushvalue (lua_State *L, int idx) { lua_lock(L); - setobj2s(L, L->top, index2value(L, idx)); + setobj2s(L, L->top.p, index2value(L, idx)); api_incr_top(L); lua_unlock(L); } @@ -342,12 +338,12 @@ LUA_API void lua_arith (lua_State *L, int op) { api_checknelems(L, 2); /* all other operations expect two operands */ else { /* for unary operations, add fake 2nd operand */ api_checknelems(L, 1); - setobjs2s(L, L->top, L->top - 1); + setobjs2s(L, L->top.p, L->top.p - 1); api_incr_top(L); } /* first operand at top - 2, second at top - 1; result go to top - 2 */ - luaO_arith(L, op, s2v(L->top - 2), s2v(L->top - 1), L->top - 2); - L->top--; /* remove second operand */ + luaO_arith(L, op, s2v(L->top.p - 2), s2v(L->top.p - 1), L->top.p - 2); + L->top.p--; /* remove second operand */ lua_unlock(L); } @@ -373,7 +369,7 @@ LUA_API int lua_compare (lua_State *L, int index1, int index2, int op) { LUA_API size_t lua_stringtonumber (lua_State *L, const char *s) { - size_t sz = luaO_str2num(s, s2v(L->top)); + size_t sz = luaO_str2num(s, s2v(L->top.p)); if (sz != 0) api_incr_top(L); return sz; @@ -500,7 +496,7 @@ LUA_API const void *lua_topointer (lua_State *L, int idx) { LUA_API void lua_pushnil (lua_State *L) { lua_lock(L); - setnilvalue(s2v(L->top)); + setnilvalue(s2v(L->top.p)); api_incr_top(L); lua_unlock(L); } @@ -508,7 +504,7 @@ LUA_API void lua_pushnil (lua_State *L) { LUA_API void lua_pushnumber (lua_State *L, lua_Number n) { lua_lock(L); - setfltvalue(s2v(L->top), n); + setfltvalue(s2v(L->top.p), n); api_incr_top(L); lua_unlock(L); } @@ -516,7 +512,7 @@ LUA_API void lua_pushnumber (lua_State *L, lua_Number n) { LUA_API void lua_pushinteger (lua_State *L, lua_Integer n) { lua_lock(L); - setivalue(s2v(L->top), n); + setivalue(s2v(L->top.p), n); api_incr_top(L); lua_unlock(L); } @@ -531,7 +527,7 @@ LUA_API const char *lua_pushlstring (lua_State *L, const char *s, size_t len) { TString *ts; lua_lock(L); ts = (len == 0) ? luaS_new(L, "") : luaS_newlstr(L, s, len); - setsvalue2s(L, L->top, ts); + setsvalue2s(L, L->top.p, ts); api_incr_top(L); luaC_checkGC(L); lua_unlock(L); @@ -542,11 +538,11 @@ LUA_API const char *lua_pushlstring (lua_State *L, const char *s, size_t len) { LUA_API const char *lua_pushstring (lua_State *L, const char *s) { lua_lock(L); if (s == NULL) - setnilvalue(s2v(L->top)); + setnilvalue(s2v(L->top.p)); else { TString *ts; ts = luaS_new(L, s); - setsvalue2s(L, L->top, ts); + setsvalue2s(L, L->top.p, ts); s = getstr(ts); /* internal copy's address */ } api_incr_top(L); @@ -583,7 +579,7 @@ LUA_API const char *lua_pushfstring (lua_State *L, const char *fmt, ...) { LUA_API void lua_pushcclosure (lua_State *L, lua_CFunction fn, int n) { lua_lock(L); if (n == 0) { - setfvalue(s2v(L->top), fn); + setfvalue(s2v(L->top.p), fn); api_incr_top(L); } else { @@ -592,13 +588,13 @@ LUA_API void lua_pushcclosure (lua_State *L, lua_CFunction fn, int n) { api_check(L, n <= MAXUPVAL, "upvalue index too large"); cl = luaF_newCclosure(L, n); cl->f = fn; - L->top -= n; + L->top.p -= n; while (n--) { - setobj2n(L, &cl->upvalue[n], s2v(L->top + n)); + setobj2n(L, &cl->upvalue[n], s2v(L->top.p + n)); /* does not need barrier because closure is white */ lua_assert(iswhite(cl)); } - setclCvalue(L, s2v(L->top), cl); + setclCvalue(L, s2v(L->top.p), cl); api_incr_top(L); luaC_checkGC(L); } @@ -609,9 +605,9 @@ LUA_API void lua_pushcclosure (lua_State *L, lua_CFunction fn, int n) { LUA_API void lua_pushboolean (lua_State *L, int b) { lua_lock(L); if (b) - setbtvalue(s2v(L->top)); + setbtvalue(s2v(L->top.p)); else - setbfvalue(s2v(L->top)); + setbfvalue(s2v(L->top.p)); api_incr_top(L); lua_unlock(L); } @@ -619,7 +615,7 @@ LUA_API void lua_pushboolean (lua_State *L, int b) { LUA_API void lua_pushlightuserdata (lua_State *L, void *p) { lua_lock(L); - setpvalue(s2v(L->top), p); + setpvalue(s2v(L->top.p), p); api_incr_top(L); lua_unlock(L); } @@ -627,7 +623,7 @@ LUA_API void lua_pushlightuserdata (lua_State *L, void *p) { LUA_API int lua_pushthread (lua_State *L) { lua_lock(L); - setthvalue(L, s2v(L->top), L); + setthvalue(L, s2v(L->top.p), L); api_incr_top(L); lua_unlock(L); return (G(L)->mainthread == L); @@ -644,16 +640,16 @@ l_sinline int auxgetstr (lua_State *L, const TValue *t, const char *k) { const TValue *slot; TString *str = luaS_new(L, k); if (luaV_fastget(L, t, str, slot, luaH_getstr)) { - setobj2s(L, L->top, slot); + setobj2s(L, L->top.p, slot); api_incr_top(L); } else { - setsvalue2s(L, L->top, str); + setsvalue2s(L, L->top.p, str); api_incr_top(L); - luaV_finishget(L, t, s2v(L->top - 1), L->top - 1, slot); + luaV_finishget(L, t, s2v(L->top.p - 1), L->top.p - 1, slot); } lua_unlock(L); - return ttype(s2v(L->top - 1)); + return ttype(s2v(L->top.p - 1)); } @@ -680,13 +676,13 @@ LUA_API int lua_gettable (lua_State *L, int idx) { TValue *t; lua_lock(L); t = index2value(L, idx); - if (luaV_fastget(L, t, s2v(L->top - 1), slot, luaH_get)) { - setobj2s(L, L->top - 1, slot); + if (luaV_fastget(L, t, s2v(L->top.p - 1), slot, luaH_get)) { + setobj2s(L, L->top.p - 1, slot); } else - luaV_finishget(L, t, s2v(L->top - 1), L->top - 1, slot); + luaV_finishget(L, t, s2v(L->top.p - 1), L->top.p - 1, slot); lua_unlock(L); - return ttype(s2v(L->top - 1)); + return ttype(s2v(L->top.p - 1)); } @@ -702,27 +698,27 @@ LUA_API int lua_geti (lua_State *L, int idx, lua_Integer n) { lua_lock(L); t = index2value(L, idx); if (luaV_fastgeti(L, t, n, slot)) { - setobj2s(L, L->top, slot); + setobj2s(L, L->top.p, slot); } else { TValue aux; setivalue(&aux, n); - luaV_finishget(L, t, &aux, L->top, slot); + luaV_finishget(L, t, &aux, L->top.p, slot); } api_incr_top(L); lua_unlock(L); - return ttype(s2v(L->top - 1)); + return ttype(s2v(L->top.p - 1)); } l_sinline int finishrawget (lua_State *L, const TValue *val) { if (isempty(val)) /* avoid copying empty items to the stack */ - setnilvalue(s2v(L->top)); + setnilvalue(s2v(L->top.p)); else - setobj2s(L, L->top, val); + setobj2s(L, L->top.p, val); api_incr_top(L); lua_unlock(L); - return ttype(s2v(L->top - 1)); + return ttype(s2v(L->top.p - 1)); } @@ -739,8 +735,8 @@ LUA_API int lua_rawget (lua_State *L, int idx) { lua_lock(L); api_checknelems(L, 1); t = gettable(L, idx); - val = luaH_get(t, s2v(L->top - 1)); - L->top--; /* remove key */ + val = luaH_get(t, s2v(L->top.p - 1)); + L->top.p--; /* remove key */ return finishrawget(L, val); } @@ -767,7 +763,7 @@ LUA_API void lua_createtable (lua_State *L, int narray, int nrec) { Table *t; lua_lock(L); t = luaH_new(L); - sethvalue2s(L, L->top, t); + sethvalue2s(L, L->top.p, t); api_incr_top(L); if (narray > 0 || nrec > 0) luaH_resize(L, t, narray, nrec); @@ -794,7 +790,7 @@ LUA_API int lua_getmetatable (lua_State *L, int objindex) { break; } if (mt != NULL) { - sethvalue2s(L, L->top, mt); + sethvalue2s(L, L->top.p, mt); api_incr_top(L); res = 1; } @@ -810,12 +806,12 @@ LUA_API int lua_getiuservalue (lua_State *L, int idx, int n) { o = index2value(L, idx); api_check(L, ttisfulluserdata(o), "full userdata expected"); if (n <= 0 || n > uvalue(o)->nuvalue) { - setnilvalue(s2v(L->top)); + setnilvalue(s2v(L->top.p)); t = LUA_TNONE; } else { - setobj2s(L, L->top, &uvalue(o)->uv[n - 1].uv); - t = ttype(s2v(L->top)); + setobj2s(L, L->top.p, &uvalue(o)->uv[n - 1].uv); + t = ttype(s2v(L->top.p)); } api_incr_top(L); lua_unlock(L); @@ -835,14 +831,14 @@ static void auxsetstr (lua_State *L, const TValue *t, const char *k) { TString *str = luaS_new(L, k); api_checknelems(L, 1); if (luaV_fastget(L, t, str, slot, luaH_getstr)) { - luaV_finishfastset(L, t, slot, s2v(L->top - 1)); - L->top--; /* pop value */ + luaV_finishfastset(L, t, slot, s2v(L->top.p - 1)); + L->top.p--; /* pop value */ } else { - setsvalue2s(L, L->top, str); /* push 'str' (to make it a TValue) */ + setsvalue2s(L, L->top.p, str); /* push 'str' (to make it a TValue) */ api_incr_top(L); - luaV_finishset(L, t, s2v(L->top - 1), s2v(L->top - 2), slot); - L->top -= 2; /* pop value and key */ + luaV_finishset(L, t, s2v(L->top.p - 1), s2v(L->top.p - 2), slot); + L->top.p -= 2; /* pop value and key */ } lua_unlock(L); /* lock done by caller */ } @@ -862,12 +858,12 @@ LUA_API void lua_settable (lua_State *L, int idx) { lua_lock(L); api_checknelems(L, 2); t = index2value(L, idx); - if (luaV_fastget(L, t, s2v(L->top - 2), slot, luaH_get)) { - luaV_finishfastset(L, t, slot, s2v(L->top - 1)); + if (luaV_fastget(L, t, s2v(L->top.p - 2), slot, luaH_get)) { + luaV_finishfastset(L, t, slot, s2v(L->top.p - 1)); } else - luaV_finishset(L, t, s2v(L->top - 2), s2v(L->top - 1), slot); - L->top -= 2; /* pop index and value */ + luaV_finishset(L, t, s2v(L->top.p - 2), s2v(L->top.p - 1), slot); + L->top.p -= 2; /* pop index and value */ lua_unlock(L); } @@ -885,14 +881,14 @@ LUA_API void lua_seti (lua_State *L, int idx, lua_Integer n) { api_checknelems(L, 1); t = index2value(L, idx); if (luaV_fastgeti(L, t, n, slot)) { - luaV_finishfastset(L, t, slot, s2v(L->top - 1)); + luaV_finishfastset(L, t, slot, s2v(L->top.p - 1)); } else { TValue aux; setivalue(&aux, n); - luaV_finishset(L, t, &aux, s2v(L->top - 1), slot); + luaV_finishset(L, t, &aux, s2v(L->top.p - 1), slot); } - L->top--; /* pop value */ + L->top.p--; /* pop value */ lua_unlock(L); } @@ -902,16 +898,16 @@ static void aux_rawset (lua_State *L, int idx, TValue *key, int n) { lua_lock(L); api_checknelems(L, n); t = gettable(L, idx); - luaH_set(L, t, key, s2v(L->top - 1)); + luaH_set(L, t, key, s2v(L->top.p - 1)); invalidateTMcache(t); - luaC_barrierback(L, obj2gco(t), s2v(L->top - 1)); - L->top -= n; + luaC_barrierback(L, obj2gco(t), s2v(L->top.p - 1)); + L->top.p -= n; lua_unlock(L); } LUA_API void lua_rawset (lua_State *L, int idx) { - aux_rawset(L, idx, s2v(L->top - 2), 2); + aux_rawset(L, idx, s2v(L->top.p - 2), 2); } @@ -927,9 +923,9 @@ LUA_API void lua_rawseti (lua_State *L, int idx, lua_Integer n) { lua_lock(L); api_checknelems(L, 1); t = gettable(L, idx); - luaH_setint(L, t, n, s2v(L->top - 1)); - luaC_barrierback(L, obj2gco(t), s2v(L->top - 1)); - L->top--; + luaH_setint(L, t, n, s2v(L->top.p - 1)); + luaC_barrierback(L, obj2gco(t), s2v(L->top.p - 1)); + L->top.p--; lua_unlock(L); } @@ -940,11 +936,11 @@ LUA_API int lua_setmetatable (lua_State *L, int objindex) { lua_lock(L); api_checknelems(L, 1); obj = index2value(L, objindex); - if (ttisnil(s2v(L->top - 1))) + if (ttisnil(s2v(L->top.p - 1))) mt = NULL; else { - api_check(L, ttistable(s2v(L->top - 1)), "table expected"); - mt = hvalue(s2v(L->top - 1)); + api_check(L, ttistable(s2v(L->top.p - 1)), "table expected"); + mt = hvalue(s2v(L->top.p - 1)); } switch (ttype(obj)) { case LUA_TTABLE: { @@ -968,7 +964,7 @@ LUA_API int lua_setmetatable (lua_State *L, int objindex) { break; } } - L->top--; + L->top.p--; lua_unlock(L); return 1; } @@ -984,11 +980,11 @@ LUA_API int lua_setiuservalue (lua_State *L, int idx, int n) { if (!(cast_uint(n) - 1u < cast_uint(uvalue(o)->nuvalue))) res = 0; /* 'n' not in [1, uvalue(o)->nuvalue] */ else { - setobj(L, &uvalue(o)->uv[n - 1].uv, s2v(L->top - 1)); - luaC_barrierback(L, gcvalue(o), s2v(L->top - 1)); + setobj(L, &uvalue(o)->uv[n - 1].uv, s2v(L->top.p - 1)); + luaC_barrierback(L, gcvalue(o), s2v(L->top.p - 1)); res = 1; } - L->top--; + L->top.p--; lua_unlock(L); return res; } @@ -1000,7 +996,8 @@ LUA_API int lua_setiuservalue (lua_State *L, int idx, int n) { #define checkresults(L,na,nr) \ - api_check(L, (nr) == LUA_MULTRET || (L->ci->top - L->top >= (nr) - (na)), \ + api_check(L, (nr) == LUA_MULTRET \ + || (L->ci->top.p - L->top.p >= (nr) - (na)), \ "results from function overflow current stack size") @@ -1013,7 +1010,7 @@ LUA_API void lua_callk (lua_State *L, int nargs, int nresults, api_checknelems(L, nargs+1); api_check(L, L->status == LUA_OK, "cannot do calls on non-normal thread"); checkresults(L, nargs, nresults); - func = L->top - (nargs+1); + func = L->top.p - (nargs+1); if (k != NULL && yieldable(L)) { /* need to prepare continuation? */ L->ci->u.c.k = k; /* save continuation */ L->ci->u.c.ctx = ctx; /* save context */ @@ -1061,7 +1058,7 @@ LUA_API int lua_pcallk (lua_State *L, int nargs, int nresults, int errfunc, api_check(L, ttisfunction(s2v(o)), "error handler must be a function"); func = savestack(L, o); } - c.func = L->top - (nargs+1); /* function to be called */ + c.func = L->top.p - (nargs+1); /* function to be called */ if (k == NULL || !yieldable(L)) { /* no continuation or no yieldable? */ c.nresults = nresults; /* do a 'conventional' protected call */ status = luaD_pcall(L, f_call, &c, savestack(L, c.func), func); @@ -1096,12 +1093,12 @@ LUA_API int lua_load (lua_State *L, lua_Reader reader, void *data, luaZ_init(L, &z, reader, data); status = luaD_protectedparser(L, &z, chunkname, mode); if (status == LUA_OK) { /* no errors? */ - LClosure *f = clLvalue(s2v(L->top - 1)); /* get newly created function */ + LClosure *f = clLvalue(s2v(L->top.p - 1)); /* get new function */ if (f->nupvalues >= 1) { /* does it have an upvalue? */ /* get global table from registry */ const TValue *gt = getGtable(L); /* set global table as 1st upvalue of 'f' (may be LUA_ENV) */ - setobj(L, f->upvals[0]->v, gt); + setobj(L, f->upvals[0]->v.p, gt); luaC_barrier(L, f->upvals[0], gt); } } @@ -1115,7 +1112,7 @@ LUA_API int lua_dump (lua_State *L, lua_Writer writer, void *data, int strip) { TValue *o; lua_lock(L); api_checknelems(L, 1); - o = s2v(L->top - 1); + o = s2v(L->top.p - 1); if (isLfunction(o)) status = luaU_dump(L, getproto(o), writer, data, strip); else @@ -1241,7 +1238,7 @@ LUA_API int lua_gc (lua_State *L, int what, ...) { LUA_API int lua_error (lua_State *L) { TValue *errobj; lua_lock(L); - errobj = s2v(L->top - 1); + errobj = s2v(L->top.p - 1); api_checknelems(L, 1); /* error object is the memory error message? */ if (ttisshrstring(errobj) && eqshrstr(tsvalue(errobj), G(L)->memerrmsg)) @@ -1259,12 +1256,12 @@ LUA_API int lua_next (lua_State *L, int idx) { lua_lock(L); api_checknelems(L, 1); t = gettable(L, idx); - more = luaH_next(L, t, L->top - 1); + more = luaH_next(L, t, L->top.p - 1); if (more) { api_incr_top(L); } else /* no more elements */ - L->top -= 1; /* remove key */ + L->top.p -= 1; /* remove key */ lua_unlock(L); return more; } @@ -1276,7 +1273,7 @@ LUA_API void lua_toclose (lua_State *L, int idx) { lua_lock(L); o = index2stack(L, idx); nresults = L->ci->nresults; - api_check(L, L->tbclist < o, "given index below or equal a marked one"); + api_check(L, L->tbclist.p < o, "given index below or equal a marked one"); luaF_newtbcupval(L, o); /* create new to-be-closed upvalue */ if (!hastocloseCfunc(nresults)) /* function not marked yet? */ L->ci->nresults = codeNresults(nresults); /* mark it */ @@ -1291,7 +1288,7 @@ LUA_API void lua_concat (lua_State *L, int n) { if (n > 0) luaV_concat(L, n); else { /* nothing to concatenate */ - setsvalue2s(L, L->top, luaS_newlstr(L, "", 0)); /* push empty string */ + setsvalue2s(L, L->top.p, luaS_newlstr(L, "", 0)); /* push empty string */ api_incr_top(L); } luaC_checkGC(L); @@ -1303,7 +1300,7 @@ LUA_API void lua_len (lua_State *L, int idx) { TValue *t; lua_lock(L); t = index2value(L, idx); - luaV_objlen(L, L->top, t); + luaV_objlen(L, L->top.p, t); api_incr_top(L); lua_unlock(L); } @@ -1348,7 +1345,7 @@ LUA_API void *lua_newuserdatauv (lua_State *L, size_t size, int nuvalue) { lua_lock(L); api_check(L, 0 <= nuvalue && nuvalue < USHRT_MAX, "invalid value"); u = luaS_newudata(L, size, nuvalue); - setuvalue(L, s2v(L->top), u); + setuvalue(L, s2v(L->top.p), u); api_incr_top(L); luaC_checkGC(L); lua_unlock(L); @@ -1374,7 +1371,7 @@ static const char *aux_upvalue (TValue *fi, int n, TValue **val, Proto *p = f->p; if (!(cast_uint(n) - 1u < cast_uint(p->sizeupvalues))) return NULL; /* 'n' not in [1, p->sizeupvalues] */ - *val = f->upvals[n-1]->v; + *val = f->upvals[n-1]->v.p; if (owner) *owner = obj2gco(f->upvals[n - 1]); name = p->upvalues[n-1].name; return (name == NULL) ? "(no name)" : getstr(name); @@ -1390,7 +1387,7 @@ LUA_API const char *lua_getupvalue (lua_State *L, int funcindex, int n) { lua_lock(L); name = aux_upvalue(index2value(L, funcindex), n, &val, NULL); if (name) { - setobj2s(L, L->top, val); + setobj2s(L, L->top.p, val); api_incr_top(L); } lua_unlock(L); @@ -1408,8 +1405,8 @@ LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n) { api_checknelems(L, 1); name = aux_upvalue(fi, n, &val, &owner); if (name) { - L->top--; - setobj(L, val, s2v(L->top)); + L->top.p--; + setobj(L, val, s2v(L->top.p)); luaC_barrier(L, owner, val); } lua_unlock(L); diff --git a/src/lua/lapi.h b/src/lua/lapi.h index 9e99cc448..a742427cd 100644 --- a/src/lua/lapi.h +++ b/src/lua/lapi.h @@ -12,23 +12,26 @@ #include "lstate.h" -/* Increments 'L->top', checking for stack overflows */ -#define api_incr_top(L) {L->top++; api_check(L, L->top <= L->ci->top, \ - "stack overflow");} +/* Increments 'L->top.p', checking for stack overflows */ +#define api_incr_top(L) {L->top.p++; \ + api_check(L, L->top.p <= L->ci->top.p, \ + "stack overflow");} /* ** If a call returns too many multiple returns, the callee may not have ** stack space to accommodate all results. In this case, this macro -** increases its stack space ('L->ci->top'). +** increases its stack space ('L->ci->top.p'). */ #define adjustresults(L,nres) \ - { if ((nres) <= LUA_MULTRET && L->ci->top < L->top) L->ci->top = L->top; } + { if ((nres) <= LUA_MULTRET && L->ci->top.p < L->top.p) \ + L->ci->top.p = L->top.p; } /* Ensure the stack has at least 'n' elements */ -#define api_checknelems(L,n) api_check(L, (n) < (L->top - L->ci->func), \ - "not enough elements in the stack") +#define api_checknelems(L,n) \ + api_check(L, (n) < (L->top.p - L->ci->func.p), \ + "not enough elements in the stack") /* diff --git a/src/lua/lauxlib.c b/src/lua/lauxlib.c index 8ed1da112..4ca6c6548 100644 --- a/src/lua/lauxlib.c +++ b/src/lua/lauxlib.c @@ -526,13 +526,14 @@ static void newbox (lua_State *L) { /* ** Compute new size for buffer 'B', enough to accommodate extra 'sz' -** bytes. +** bytes. (The test for "not big enough" also gets the case when the +** computation of 'newsize' overflows.) */ static size_t newbuffsize (luaL_Buffer *B, size_t sz) { - size_t newsize = B->size * 2; /* double buffer size */ + size_t newsize = (B->size / 2) * 3; /* buffer size * 1.5 */ if (l_unlikely(MAX_SIZET - sz < B->n)) /* overflow in (B->n + sz)? */ return luaL_error(B->L, "buffer too large"); - if (newsize < B->n + sz) /* double is not big enough? */ + if (newsize < B->n + sz) /* not big enough? */ newsize = B->n + sz; return newsize; } @@ -611,7 +612,7 @@ LUALIB_API void luaL_pushresultsize (luaL_Buffer *B, size_t sz) { ** box (if existent) is not on the top of the stack. So, instead of ** calling 'luaL_addlstring', it replicates the code using -2 as the ** last argument to 'prepbuffsize', signaling that the box is (or will -** be) bellow the string being added to the buffer. (Box creation can +** be) below the string being added to the buffer. (Box creation can ** trigger an emergency GC, so we should not remove the string from the ** stack before we have the space guaranteed.) */ @@ -739,17 +740,18 @@ static int errfile (lua_State *L, const char *what, int fnameindex) { } -static int skipBOM (LoadF *lf) { - const char *p = "\xEF\xBB\xBF"; /* UTF-8 BOM mark */ - int c; - lf->n = 0; - do { - c = getc(lf->f); - if (c == EOF || c != *(const unsigned char *)p++) return c; - lf->buff[lf->n++] = c; /* to be read by the parser */ - } while (*p != '\0'); - lf->n = 0; /* prefix matched; discard it */ - return getc(lf->f); /* return next character */ +/* +** Skip an optional BOM at the start of a stream. If there is an +** incomplete BOM (the first character is correct but the rest is +** not), returns the first character anyway to force an error +** (as no chunk can start with 0xEF). +*/ +static int skipBOM (FILE *f) { + int c = getc(f); /* read first character */ + if (c == 0xEF && getc(f) == 0xBB && getc(f) == 0xBF) /* correct BOM? */ + return getc(f); /* ignore BOM and return next char */ + else /* no (valid) BOM */ + return c; /* return first character */ } @@ -760,13 +762,13 @@ static int skipBOM (LoadF *lf) { ** first "valid" character of the file (after the optional BOM and ** a first-line comment). */ -static int skipcomment (LoadF *lf, int *cp) { - int c = *cp = skipBOM(lf); +static int skipcomment (FILE *f, int *cp) { + int c = *cp = skipBOM(f); if (c == '#') { /* first line is a comment (Unix exec. file)? */ do { /* skip first line */ - c = getc(lf->f); + c = getc(f); } while (c != EOF && c != '\n'); - *cp = getc(lf->f); /* skip end-of-line, if present */ + *cp = getc(f); /* next character after comment, if present */ return 1; /* there was a comment */ } else return 0; /* no comment */ @@ -788,12 +790,16 @@ LUALIB_API int luaL_loadfilex (lua_State *L, const char *filename, lf.f = fopen(filename, "r"); if (lf.f == NULL) return errfile(L, "open", fnameindex); } - if (skipcomment(&lf, &c)) /* read initial portion */ - lf.buff[lf.n++] = '\n'; /* add line to correct line numbers */ - if (c == LUA_SIGNATURE[0] && filename) { /* binary file? */ - lf.f = freopen(filename, "rb", lf.f); /* reopen in binary mode */ - if (lf.f == NULL) return errfile(L, "reopen", fnameindex); - skipcomment(&lf, &c); /* re-read initial portion */ + lf.n = 0; + if (skipcomment(lf.f, &c)) /* read initial portion */ + lf.buff[lf.n++] = '\n'; /* add newline to correct line numbers */ + if (c == LUA_SIGNATURE[0]) { /* binary file? */ + lf.n = 0; /* remove possible newline */ + if (filename) { /* "real" file? */ + lf.f = freopen(filename, "rb", lf.f); /* reopen in binary mode */ + if (lf.f == NULL) return errfile(L, "reopen", fnameindex); + skipcomment(lf.f, &c); /* re-read initial portion */ + } } if (c != EOF) lf.buff[lf.n++] = c; /* 'c' is the first character of the stream */ diff --git a/src/lua/lcode.c b/src/lua/lcode.c index 06425a1db..1a371ca94 100644 --- a/src/lua/lcode.c +++ b/src/lua/lcode.c @@ -1351,6 +1351,35 @@ static int constfolding (FuncState *fs, int op, expdesc *e1, } +/* +** Convert a BinOpr to an OpCode (ORDER OPR - ORDER OP) +*/ +l_sinline OpCode binopr2op (BinOpr opr, BinOpr baser, OpCode base) { + lua_assert(baser <= opr && + ((baser == OPR_ADD && opr <= OPR_SHR) || + (baser == OPR_LT && opr <= OPR_LE))); + return cast(OpCode, (cast_int(opr) - cast_int(baser)) + cast_int(base)); +} + + +/* +** Convert a UnOpr to an OpCode (ORDER OPR - ORDER OP) +*/ +l_sinline OpCode unopr2op (UnOpr opr) { + return cast(OpCode, (cast_int(opr) - cast_int(OPR_MINUS)) + + cast_int(OP_UNM)); +} + + +/* +** Convert a BinOpr to a tag method (ORDER OPR - ORDER TM) +*/ +l_sinline TMS binopr2TM (BinOpr opr) { + lua_assert(OPR_ADD <= opr && opr <= OPR_SHR); + return cast(TMS, (cast_int(opr) - cast_int(OPR_ADD)) + cast_int(TM_ADD)); +} + + /* ** Emit code for unary expressions that "produce values" ** (everything but 'not'). @@ -1389,12 +1418,15 @@ static void finishbinexpval (FuncState *fs, expdesc *e1, expdesc *e2, ** Emit code for binary expressions that "produce values" over ** two registers. */ -static void codebinexpval (FuncState *fs, OpCode op, +static void codebinexpval (FuncState *fs, BinOpr opr, expdesc *e1, expdesc *e2, int line) { - int v2 = luaK_exp2anyreg(fs, e2); /* both operands are in registers */ + OpCode op = binopr2op(opr, OPR_ADD, OP_ADD); + int v2 = luaK_exp2anyreg(fs, e2); /* make sure 'e2' is in a register */ + /* 'e1' must be already in a register or it is a constant */ + lua_assert((VNIL <= e1->k && e1->k <= VKSTR) || + e1->k == VNONRELOC || e1->k == VRELOC); lua_assert(OP_ADD <= op && op <= OP_SHR); - finishbinexpval(fs, e1, e2, op, v2, 0, line, OP_MMBIN, - cast(TMS, (op - OP_ADD) + TM_ADD)); + finishbinexpval(fs, e1, e2, op, v2, 0, line, OP_MMBIN, binopr2TM(opr)); } @@ -1410,6 +1442,18 @@ static void codebini (FuncState *fs, OpCode op, } +/* +** Code binary operators with K operand. +*/ +static void codebinK (FuncState *fs, BinOpr opr, + expdesc *e1, expdesc *e2, int flip, int line) { + TMS event = binopr2TM(opr); + int v2 = e2->u.info; /* K index */ + OpCode op = binopr2op(opr, OPR_ADD, OP_ADDK); + finishbinexpval(fs, e1, e2, op, v2, flip, line, OP_MMBINK, event); +} + + /* Try to code a binary operator negating its second operand. ** For the metamethod, 2nd operand must keep its original value. */ @@ -1437,24 +1481,27 @@ static void swapexps (expdesc *e1, expdesc *e2) { } +/* +** Code binary operators with no constant operand. +*/ +static void codebinNoK (FuncState *fs, BinOpr opr, + expdesc *e1, expdesc *e2, int flip, int line) { + if (flip) + swapexps(e1, e2); /* back to original order */ + codebinexpval(fs, opr, e1, e2, line); /* use standard operators */ +} + + /* ** Code arithmetic operators ('+', '-', ...). If second operand is a ** constant in the proper range, use variant opcodes with K operands. */ static void codearith (FuncState *fs, BinOpr opr, expdesc *e1, expdesc *e2, int flip, int line) { - TMS event = cast(TMS, opr + TM_ADD); - if (tonumeral(e2, NULL) && luaK_exp2K(fs, e2)) { /* K operand? */ - int v2 = e2->u.info; /* K index */ - OpCode op = cast(OpCode, opr + OP_ADDK); - finishbinexpval(fs, e1, e2, op, v2, flip, line, OP_MMBINK, event); - } - else { /* 'e2' is neither an immediate nor a K operand */ - OpCode op = cast(OpCode, opr + OP_ADD); - if (flip) - swapexps(e1, e2); /* back to original order */ - codebinexpval(fs, op, e1, e2, line); /* use standard operators */ - } + if (tonumeral(e2, NULL) && luaK_exp2K(fs, e2)) /* K operand? */ + codebinK(fs, opr, e1, e2, flip, line); + else /* 'e2' is neither an immediate nor a K operand */ + codebinNoK(fs, opr, e1, e2, flip, line); } @@ -1471,35 +1518,27 @@ static void codecommutative (FuncState *fs, BinOpr op, flip = 1; } if (op == OPR_ADD && isSCint(e2)) /* immediate operand? */ - codebini(fs, cast(OpCode, OP_ADDI), e1, e2, flip, line, TM_ADD); + codebini(fs, OP_ADDI, e1, e2, flip, line, TM_ADD); else codearith(fs, op, e1, e2, flip, line); } /* -** Code bitwise operations; they are all associative, so the function +** Code bitwise operations; they are all commutative, so the function ** tries to put an integer constant as the 2nd operand (a K operand). */ static void codebitwise (FuncState *fs, BinOpr opr, expdesc *e1, expdesc *e2, int line) { int flip = 0; - int v2; - OpCode op; - if (e1->k == VKINT && luaK_exp2RK(fs, e1)) { + if (e1->k == VKINT) { swapexps(e1, e2); /* 'e2' will be the constant operand */ flip = 1; } - else if (!(e2->k == VKINT && luaK_exp2RK(fs, e2))) { /* no constants? */ - op = cast(OpCode, opr + OP_ADD); - codebinexpval(fs, op, e1, e2, line); /* all-register opcodes */ - return; - } - v2 = e2->u.info; /* index in K array */ - op = cast(OpCode, opr + OP_ADDK); - lua_assert(ttisinteger(&fs->f->k[v2])); - finishbinexpval(fs, e1, e2, op, v2, flip, line, OP_MMBINK, - cast(TMS, opr + TM_ADD)); + if (e2->k == VKINT && luaK_exp2K(fs, e2)) /* K operand? */ + codebinK(fs, opr, e1, e2, flip, line); + else /* no constants */ + codebinNoK(fs, opr, e1, e2, flip, line); } @@ -1507,25 +1546,27 @@ static void codebitwise (FuncState *fs, BinOpr opr, ** Emit code for order comparisons. When using an immediate operand, ** 'isfloat' tells whether the original value was a float. */ -static void codeorder (FuncState *fs, OpCode op, expdesc *e1, expdesc *e2) { +static void codeorder (FuncState *fs, BinOpr opr, expdesc *e1, expdesc *e2) { int r1, r2; int im; int isfloat = 0; + OpCode op; if (isSCnumber(e2, &im, &isfloat)) { /* use immediate operand */ r1 = luaK_exp2anyreg(fs, e1); r2 = im; - op = cast(OpCode, (op - OP_LT) + OP_LTI); + op = binopr2op(opr, OPR_LT, OP_LTI); } else if (isSCnumber(e1, &im, &isfloat)) { /* transform (A < B) to (B > A) and (A <= B) to (B >= A) */ r1 = luaK_exp2anyreg(fs, e2); r2 = im; - op = (op == OP_LT) ? OP_GTI : OP_GEI; + op = binopr2op(opr, OPR_LT, OP_GTI); } else { /* regular case, compare two registers */ r1 = luaK_exp2anyreg(fs, e1); r2 = luaK_exp2anyreg(fs, e2); + op = binopr2op(opr, OPR_LT, OP_LT); } freeexps(fs, e1, e2); e1->u.info = condjump(fs, op, r1, r2, isfloat, 1); @@ -1551,7 +1592,7 @@ static void codeeq (FuncState *fs, BinOpr opr, expdesc *e1, expdesc *e2) { op = OP_EQI; r2 = im; /* immediate operand */ } - else if (luaK_exp2RK(fs, e2)) { /* 1st expression is constant? */ + else if (luaK_exp2RK(fs, e2)) { /* 2nd expression is constant? */ op = OP_EQK; r2 = e2->u.info; /* constant index */ } @@ -1568,16 +1609,16 @@ static void codeeq (FuncState *fs, BinOpr opr, expdesc *e1, expdesc *e2) { /* ** Apply prefix operation 'op' to expression 'e'. */ -void luaK_prefix (FuncState *fs, UnOpr op, expdesc *e, int line) { +void luaK_prefix (FuncState *fs, UnOpr opr, expdesc *e, int line) { static const expdesc ef = {VKINT, {0}, NO_JUMP, NO_JUMP}; luaK_dischargevars(fs, e); - switch (op) { + switch (opr) { case OPR_MINUS: case OPR_BNOT: /* use 'ef' as fake 2nd operand */ - if (constfolding(fs, op + LUA_OPUNM, e, &ef)) + if (constfolding(fs, opr + LUA_OPUNM, e, &ef)) break; /* else */ /* FALLTHROUGH */ case OPR_LEN: - codeunexpval(fs, cast(OpCode, op + OP_UNM), e, line); + codeunexpval(fs, unopr2op(opr), e, line); break; case OPR_NOT: codenot(fs, e); break; default: lua_assert(0); @@ -1611,7 +1652,8 @@ void luaK_infix (FuncState *fs, BinOpr op, expdesc *v) { case OPR_SHL: case OPR_SHR: { if (!tonumeral(v, NULL)) luaK_exp2anyreg(fs, v); - /* else keep numeral, which may be folded with 2nd operand */ + /* else keep numeral, which may be folded or used as an immediate + operand */ break; } case OPR_EQ: case OPR_NE: { @@ -1706,30 +1748,27 @@ void luaK_posfix (FuncState *fs, BinOpr opr, /* coded as (r1 >> -I) */; } else /* regular case (two registers) */ - codebinexpval(fs, OP_SHL, e1, e2, line); + codebinexpval(fs, opr, e1, e2, line); break; } case OPR_SHR: { if (isSCint(e2)) codebini(fs, OP_SHRI, e1, e2, 0, line, TM_SHR); /* r1 >> I */ else /* regular case (two registers) */ - codebinexpval(fs, OP_SHR, e1, e2, line); + codebinexpval(fs, opr, e1, e2, line); break; } case OPR_EQ: case OPR_NE: { codeeq(fs, opr, e1, e2); break; } - case OPR_LT: case OPR_LE: { - OpCode op = cast(OpCode, (opr - OPR_EQ) + OP_EQ); - codeorder(fs, op, e1, e2); - break; - } case OPR_GT: case OPR_GE: { /* '(a > b)' <=> '(b < a)'; '(a >= b)' <=> '(b <= a)' */ - OpCode op = cast(OpCode, (opr - OPR_NE) + OP_EQ); swapexps(e1, e2); - codeorder(fs, op, e1, e2); + opr = cast(BinOpr, (opr - OPR_GT) + OPR_LT); + } /* FALLTHROUGH */ + case OPR_LT: case OPR_LE: { + codeorder(fs, opr, e1, e2); break; } default: lua_assert(0); diff --git a/src/lua/lcorolib.c b/src/lua/lcorolib.c index 785a1e81a..40b880b14 100644 --- a/src/lua/lcorolib.c +++ b/src/lua/lcorolib.c @@ -76,7 +76,7 @@ static int luaB_auxwrap (lua_State *L) { if (l_unlikely(r < 0)) { /* error? */ int stat = lua_status(co); if (stat != LUA_OK && stat != LUA_YIELD) { /* error in the coroutine? */ - stat = lua_resetthread(co); /* close its tbc variables */ + stat = lua_resetthread(co, L); /* close its tbc variables */ lua_assert(stat != LUA_OK); lua_xmove(co, L, 1); /* move error message to the caller */ } @@ -172,7 +172,7 @@ static int luaB_close (lua_State *L) { int status = auxstatus(L, co); switch (status) { case COS_DEAD: case COS_YIELD: { - status = lua_resetthread(co); + status = lua_resetthread(co, L); if (status == LUA_OK) { lua_pushboolean(L, 1); return 1; diff --git a/src/lua/ldebug.c b/src/lua/ldebug.c index a716d95e2..28b1caabf 100644 --- a/src/lua/ldebug.c +++ b/src/lua/ldebug.c @@ -182,10 +182,10 @@ static const char *upvalname (const Proto *p, int uv) { static const char *findvararg (CallInfo *ci, int n, StkId *pos) { - if (clLvalue(s2v(ci->func))->p->is_vararg) { + if (clLvalue(s2v(ci->func.p))->p->is_vararg) { int nextra = ci->u.l.nextraargs; if (n >= -nextra) { /* 'n' is negative */ - *pos = ci->func - nextra - (n + 1); + *pos = ci->func.p - nextra - (n + 1); return "(vararg)"; /* generic name for any vararg */ } } @@ -194,7 +194,7 @@ static const char *findvararg (CallInfo *ci, int n, StkId *pos) { const char *luaG_findlocal (lua_State *L, CallInfo *ci, int n, StkId *pos) { - StkId base = ci->func + 1; + StkId base = ci->func.p + 1; const char *name = NULL; if (isLua(ci)) { if (n < 0) /* access to vararg values? */ @@ -203,7 +203,7 @@ const char *luaG_findlocal (lua_State *L, CallInfo *ci, int n, StkId *pos) { name = luaF_getlocalname(ci_func(ci)->p, n, currentpc(ci)); } if (name == NULL) { /* no 'standard' name? */ - StkId limit = (ci == L->ci) ? L->top : ci->next->func; + StkId limit = (ci == L->ci) ? L->top.p : ci->next->func.p; if (limit - base >= n && n > 0) { /* is 'n' inside 'ci' stack? */ /* generic name for any valid slot */ name = isLua(ci) ? "(temporary)" : "(C temporary)"; @@ -221,16 +221,16 @@ LUA_API const char *lua_getlocal (lua_State *L, const lua_Debug *ar, int n) { const char *name; lua_lock(L); if (ar == NULL) { /* information about non-active function? */ - if (!isLfunction(s2v(L->top - 1))) /* not a Lua function? */ + if (!isLfunction(s2v(L->top.p - 1))) /* not a Lua function? */ name = NULL; else /* consider live variables at function start (parameters) */ - name = luaF_getlocalname(clLvalue(s2v(L->top - 1))->p, n, 0); + name = luaF_getlocalname(clLvalue(s2v(L->top.p - 1))->p, n, 0); } else { /* active function; get information through 'ar' */ StkId pos = NULL; /* to avoid warnings */ name = luaG_findlocal(L, ar->i_ci, n, &pos); if (name) { - setobjs2s(L, L->top, pos); + setobjs2s(L, L->top.p, pos); api_incr_top(L); } } @@ -245,8 +245,8 @@ LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n) { lua_lock(L); name = luaG_findlocal(L, ar->i_ci, n, &pos); if (name) { - setobjs2s(L, pos, L->top - 1); - L->top--; /* pop value */ + setobjs2s(L, pos, L->top.p - 1); + L->top.p--; /* pop value */ } lua_unlock(L); return name; @@ -289,7 +289,7 @@ static int nextline (const Proto *p, int currentline, int pc) { static void collectvalidlines (lua_State *L, Closure *f) { if (noLuaClosure(f)) { - setnilvalue(s2v(L->top)); + setnilvalue(s2v(L->top.p)); api_incr_top(L); } else { @@ -298,7 +298,7 @@ static void collectvalidlines (lua_State *L, Closure *f) { const Proto *p = f->l.p; int currentline = p->linedefined; Table *t = luaH_new(L); /* new table to store active lines */ - sethvalue2s(L, L->top, t); /* push it on stack */ + sethvalue2s(L, L->top.p, t); /* push it on stack */ api_incr_top(L); setbtvalue(&v); /* boolean 'true' to be the value of all indices */ if (!p->is_vararg) /* regular function? */ @@ -388,20 +388,20 @@ LUA_API int lua_getinfo (lua_State *L, const char *what, lua_Debug *ar) { lua_lock(L); if (*what == '>') { ci = NULL; - func = s2v(L->top - 1); + func = s2v(L->top.p - 1); api_check(L, ttisfunction(func), "function expected"); what++; /* skip the '>' */ - L->top--; /* pop function */ + L->top.p--; /* pop function */ } else { ci = ar->i_ci; - func = s2v(ci->func); + func = s2v(ci->func.p); lua_assert(ttisfunction(func)); } cl = ttisclosure(func) ? clvalue(func) : NULL; status = auxgetinfo(L, what, ar, cl, ci); if (strchr(what, 'f')) { - setobj2s(L, L->top, func); + setobj2s(L, L->top.p, func); api_incr_top(L); } if (strchr(what, 'L')) @@ -656,18 +656,19 @@ static const char *funcnamefromcall (lua_State *L, CallInfo *ci, /* -** Check whether pointer 'o' points to some value in the stack -** frame of the current function. Because 'o' may not point to a -** value in this stack, we cannot compare it with the region -** boundaries (undefined behaviour in ISO C). +** Check whether pointer 'o' points to some value in the stack frame of +** the current function and, if so, returns its index. Because 'o' may +** not point to a value in this stack, we cannot compare it with the +** region boundaries (undefined behavior in ISO C). */ -static int isinstack (CallInfo *ci, const TValue *o) { - StkId pos; - for (pos = ci->func + 1; pos < ci->top; pos++) { - if (o == s2v(pos)) - return 1; +static int instack (CallInfo *ci, const TValue *o) { + int pos; + StkId base = ci->func.p + 1; + for (pos = 0; base + pos < ci->top.p; pos++) { + if (o == s2v(base + pos)) + return pos; } - return 0; /* not found */ + return -1; /* not found */ } @@ -681,7 +682,7 @@ static const char *getupvalname (CallInfo *ci, const TValue *o, LClosure *c = ci_func(ci); int i; for (i = 0; i < c->nupvalues; i++) { - if (c->upvals[i]->v == o) { + if (c->upvals[i]->v.p == o) { *name = upvalname(c->p, i); return "upvalue"; } @@ -708,9 +709,11 @@ static const char *varinfo (lua_State *L, const TValue *o) { const char *kind = NULL; if (isLua(ci)) { kind = getupvalname(ci, o, &name); /* check whether 'o' is an upvalue */ - if (!kind && isinstack(ci, o)) /* no? try a register */ - kind = getobjname(ci_func(ci)->p, currentpc(ci), - cast_int(cast(StkId, o) - (ci->func + 1)), &name); + if (!kind) { /* not an upvalue? */ + int reg = instack(ci, o); /* try a register */ + if (reg >= 0) /* is 'o' a register? */ + kind = getobjname(ci_func(ci)->p, currentpc(ci), reg, &name); + } } return formatvarinfo(L, kind, name); } @@ -807,10 +810,10 @@ l_noret luaG_errormsg (lua_State *L) { if (L->errfunc != 0) { /* is there an error handling function? */ StkId errfunc = restorestack(L, L->errfunc); lua_assert(ttisfunction(s2v(errfunc))); - setobjs2s(L, L->top, L->top - 1); /* move argument */ - setobjs2s(L, L->top - 1, errfunc); /* push function */ - L->top++; /* assume EXTRA_STACK */ - luaD_callnoyield(L, L->top - 2, 1); /* call it */ + setobjs2s(L, L->top.p, L->top.p - 1); /* move argument */ + setobjs2s(L, L->top.p - 1, errfunc); /* push function */ + L->top.p++; /* assume EXTRA_STACK */ + luaD_callnoyield(L, L->top.p - 2, 1); /* call it */ } luaD_throw(L, LUA_ERRRUN); } @@ -824,8 +827,11 @@ l_noret luaG_runerror (lua_State *L, const char *fmt, ...) { va_start(argp, fmt); msg = luaO_pushvfstring(L, fmt, argp); /* format message */ va_end(argp); - if (isLua(ci)) /* if Lua function, add source:line information */ + if (isLua(ci)) { /* if Lua function, add source:line information */ luaG_addinfo(L, msg, ci_func(ci)->p->source, getcurrentline(ci)); + setobjs2s(L, L->top.p - 2, L->top.p - 1); /* remove 'msg' */ + L->top.p--; + } luaG_errormsg(L); } @@ -842,7 +848,7 @@ static int changedline (const Proto *p, int oldpc, int newpc) { if (p->lineinfo == NULL) /* no debug information? */ return 0; if (newpc - oldpc < MAXIWTHABS / 2) { /* not too far apart? */ - int delta = 0; /* line diference */ + int delta = 0; /* line difference */ int pc = oldpc; for (;;) { int lineinfo = p->lineinfo[++pc]; @@ -869,7 +875,7 @@ static int changedline (const Proto *p, int oldpc, int newpc) { ** invalid; if so, use zero as a valid value. (A wrong but valid 'oldpc' ** at most causes an extra call to a line hook.) ** This function is not "Protected" when called, so it should correct -** 'L->top' before calling anything that can run the GC. +** 'L->top.p' before calling anything that can run the GC. */ int luaG_traceexec (lua_State *L, const Instruction *pc) { CallInfo *ci = L->ci; @@ -892,7 +898,7 @@ int luaG_traceexec (lua_State *L, const Instruction *pc) { return 1; /* do not call hook again (VM yielded, so it did not move) */ } if (!isIT(*(ci->u.l.savedpc - 1))) /* top not being used? */ - L->top = ci->top; /* correct top */ + L->top.p = ci->top.p; /* correct top */ if (counthook) luaD_hook(L, LUA_HOOKCOUNT, -1, 0, 0); /* call count hook */ if (mask & LUA_MASKLINE) { diff --git a/src/lua/ldebug.h b/src/lua/ldebug.h index 974960e99..2c3074c61 100644 --- a/src/lua/ldebug.h +++ b/src/lua/ldebug.h @@ -15,7 +15,7 @@ /* Active Lua function (given call info) */ -#define ci_func(ci) (clLvalue(s2v((ci)->func))) +#define ci_func(ci) (clLvalue(s2v((ci)->func.p))) #define resethookcount(L) (L->hookcount = L->basehookcount) diff --git a/src/lua/ldo.c b/src/lua/ldo.c index a48e35f9d..2a0017ca6 100644 --- a/src/lua/ldo.c +++ b/src/lua/ldo.c @@ -104,11 +104,11 @@ void luaD_seterrorobj (lua_State *L, int errcode, StkId oldtop) { } default: { lua_assert(errorstatus(errcode)); /* real error */ - setobjs2s(L, oldtop, L->top - 1); /* error message on current top */ + setobjs2s(L, oldtop, L->top.p - 1); /* error message on current top */ break; } } - L->top = oldtop + 1; + L->top.p = oldtop + 1; } @@ -121,7 +121,7 @@ l_noret luaD_throw (lua_State *L, int errcode) { global_State *g = G(L); errcode = luaE_resetthread(L, errcode); /* close all upvalues */ if (g->mainthread->errorJmp) { /* main thread has a handler? */ - setobjs2s(L, g->mainthread->top++, L->top - 1); /* copy error obj. */ + setobjs2s(L, g->mainthread->top.p++, L->top.p - 1); /* copy error obj. */ luaD_throw(g->mainthread, errcode); /* re-throw in main thread */ } else { /* no handler at all; abort */ @@ -157,16 +157,38 @@ int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) { ** Stack reallocation ** =================================================================== */ -static void correctstack (lua_State *L, StkId oldstack, StkId newstack) { + + +/* +** Change all pointers to the stack into offsets. +*/ +static void relstack (lua_State *L) { + CallInfo *ci; + UpVal *up; + L->top.offset = savestack(L, L->top.p); + L->tbclist.offset = savestack(L, L->tbclist.p); + for (up = L->openupval; up != NULL; up = up->u.open.next) + up->v.offset = savestack(L, uplevel(up)); + for (ci = L->ci; ci != NULL; ci = ci->previous) { + ci->top.offset = savestack(L, ci->top.p); + ci->func.offset = savestack(L, ci->func.p); + } +} + + +/* +** Change back all offsets into pointers. +*/ +static void correctstack (lua_State *L) { CallInfo *ci; UpVal *up; - L->top = (L->top - oldstack) + newstack; - L->tbclist = (L->tbclist - oldstack) + newstack; + L->top.p = restorestack(L, L->top.offset); + L->tbclist.p = restorestack(L, L->tbclist.offset); for (up = L->openupval; up != NULL; up = up->u.open.next) - up->v = s2v((uplevel(up) - oldstack) + newstack); + up->v.p = s2v(restorestack(L, up->v.offset)); for (ci = L->ci; ci != NULL; ci = ci->previous) { - ci->top = (ci->top - oldstack) + newstack; - ci->func = (ci->func - oldstack) + newstack; + ci->top.p = restorestack(L, ci->top.offset); + ci->func.p = restorestack(L, ci->func.offset); if (isLua(ci)) ci->u.l.trap = 1; /* signal to update 'trap' in 'luaV_execute' */ } @@ -176,44 +198,45 @@ static void correctstack (lua_State *L, StkId oldstack, StkId newstack) { /* some space for error handling */ #define ERRORSTACKSIZE (LUAI_MAXSTACK + 200) - /* -** Reallocate the stack to a new size, correcting all pointers into -** it. (There are pointers to a stack from its upvalues, from its list -** of call infos, plus a few individual pointers.) The reallocation is -** done in two steps (allocation + free) because the correction must be -** done while both addresses (the old stack and the new one) are valid. -** (In ISO C, any pointer use after the pointer has been deallocated is -** undefined behavior.) +** Reallocate the stack to a new size, correcting all pointers into it. +** In ISO C, any pointer use after the pointer has been deallocated is +** undefined behavior. So, before the reallocation, all pointers are +** changed to offsets, and after the reallocation they are changed back +** to pointers. As during the reallocation the pointers are invalid, the +** reallocation cannot run emergency collections. +** ** In case of allocation error, raise an error or return false according ** to 'raiseerror'. */ int luaD_reallocstack (lua_State *L, int newsize, int raiseerror) { int oldsize = stacksize(L); int i; - StkId newstack = luaM_reallocvector(L, NULL, 0, - newsize + EXTRA_STACK, StackValue); + StkId newstack; + int oldgcstop = G(L)->gcstopem; lua_assert(newsize <= LUAI_MAXSTACK || newsize == ERRORSTACKSIZE); + relstack(L); /* change pointers to offsets */ + G(L)->gcstopem = 1; /* stop emergency collection */ + newstack = luaM_reallocvector(L, L->stack.p, oldsize + EXTRA_STACK, + newsize + EXTRA_STACK, StackValue); + G(L)->gcstopem = oldgcstop; /* restore emergency collection */ if (l_unlikely(newstack == NULL)) { /* reallocation failed? */ + correctstack(L); /* change offsets back to pointers */ if (raiseerror) luaM_error(L); else return 0; /* do not raise an error */ } - /* number of elements to be copied to the new stack */ - i = ((oldsize <= newsize) ? oldsize : newsize) + EXTRA_STACK; - memcpy(newstack, L->stack, i * sizeof(StackValue)); - for (; i < newsize + EXTRA_STACK; i++) + L->stack.p = newstack; + correctstack(L); /* change offsets back to pointers */ + L->stack_last.p = L->stack.p + newsize; + for (i = oldsize + EXTRA_STACK; i < newsize + EXTRA_STACK; i++) setnilvalue(s2v(newstack + i)); /* erase new segment */ - correctstack(L, L->stack, newstack); - luaM_freearray(L, L->stack, oldsize + EXTRA_STACK); - L->stack = newstack; - L->stack_last = L->stack + newsize; return 1; } /* -** Try to grow the stack by at least 'n' elements. when 'raiseerror' +** Try to grow the stack by at least 'n' elements. When 'raiseerror' ** is true, raises any error; otherwise, return 0 in case of errors. */ int luaD_growstack (lua_State *L, int n, int raiseerror) { @@ -227,35 +250,38 @@ int luaD_growstack (lua_State *L, int n, int raiseerror) { luaD_throw(L, LUA_ERRERR); /* error inside message handler */ return 0; /* if not 'raiseerror', just signal it */ } - else { + else if (n < LUAI_MAXSTACK) { /* avoids arithmetic overflows */ int newsize = 2 * size; /* tentative new size */ - int needed = cast_int(L->top - L->stack) + n; + int needed = cast_int(L->top.p - L->stack.p) + n; if (newsize > LUAI_MAXSTACK) /* cannot cross the limit */ newsize = LUAI_MAXSTACK; if (newsize < needed) /* but must respect what was asked for */ newsize = needed; if (l_likely(newsize <= LUAI_MAXSTACK)) return luaD_reallocstack(L, newsize, raiseerror); - else { /* stack overflow */ - /* add extra size to be able to handle the error message */ - luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror); - if (raiseerror) - luaG_runerror(L, "stack overflow"); - return 0; - } } + /* else stack overflow */ + /* add extra size to be able to handle the error message */ + luaD_reallocstack(L, ERRORSTACKSIZE, raiseerror); + if (raiseerror) + luaG_runerror(L, "stack overflow"); + return 0; } +/* +** Compute how much of the stack is being used, by computing the +** maximum top of all call frames in the stack and the current top. +*/ static int stackinuse (lua_State *L) { CallInfo *ci; int res; - StkId lim = L->top; + StkId lim = L->top.p; for (ci = L->ci; ci != NULL; ci = ci->previous) { - if (lim < ci->top) lim = ci->top; + if (lim < ci->top.p) lim = ci->top.p; } - lua_assert(lim <= L->stack_last); - res = cast_int(lim - L->stack) + 1; /* part of stack in use */ + lua_assert(lim <= L->stack_last.p + EXTRA_STACK); + res = cast_int(lim - L->stack.p) + 1; /* part of stack in use */ if (res < LUA_MINSTACK) res = LUA_MINSTACK; /* ensure a minimum size */ return res; @@ -273,17 +299,13 @@ static int stackinuse (lua_State *L) { */ void luaD_shrinkstack (lua_State *L) { int inuse = stackinuse(L); - int nsize = inuse * 2; /* proposed new size */ - int max = inuse * 3; /* maximum "reasonable" size */ - if (max > LUAI_MAXSTACK) { - max = LUAI_MAXSTACK; /* respect stack limit */ - if (nsize > LUAI_MAXSTACK) - nsize = LUAI_MAXSTACK; - } + int max = (inuse > LUAI_MAXSTACK / 3) ? LUAI_MAXSTACK : inuse * 3; /* if thread is currently not handling a stack overflow and its size is larger than maximum "reasonable" size, shrink it */ - if (inuse <= LUAI_MAXSTACK && stacksize(L) > max) + if (inuse <= LUAI_MAXSTACK && stacksize(L) > max) { + int nsize = (inuse > LUAI_MAXSTACK / 2) ? LUAI_MAXSTACK : inuse * 2; luaD_reallocstack(L, nsize, 0); /* ok if that fails */ + } else /* don't change stack */ condmovestack(L,{},{}); /* (change only for debugging) */ luaE_shrinkCI(L); /* shrink CI list */ @@ -292,7 +314,7 @@ void luaD_shrinkstack (lua_State *L) { void luaD_inctop (lua_State *L) { luaD_checkstack(L, 1); - L->top++; + L->top.p++; } /* }================================================================== */ @@ -309,8 +331,8 @@ void luaD_hook (lua_State *L, int event, int line, if (hook && L->allowhook) { /* make sure there is a hook */ int mask = CIST_HOOKED; CallInfo *ci = L->ci; - ptrdiff_t top = savestack(L, L->top); /* preserve original 'top' */ - ptrdiff_t ci_top = savestack(L, ci->top); /* idem for 'ci->top' */ + ptrdiff_t top = savestack(L, L->top.p); /* preserve original 'top' */ + ptrdiff_t ci_top = savestack(L, ci->top.p); /* idem for 'ci->top' */ lua_Debug ar; ar.event = event; ar.currentline = line; @@ -320,11 +342,11 @@ void luaD_hook (lua_State *L, int event, int line, ci->u2.transferinfo.ftransfer = ftransfer; ci->u2.transferinfo.ntransfer = ntransfer; } - if (isLua(ci) && L->top < ci->top) - L->top = ci->top; /* protect entire activation register */ + if (isLua(ci) && L->top.p < ci->top.p) + L->top.p = ci->top.p; /* protect entire activation register */ luaD_checkstack(L, LUA_MINSTACK); /* ensure minimum stack size */ - if (ci->top < L->top + LUA_MINSTACK) - ci->top = L->top + LUA_MINSTACK; + if (ci->top.p < L->top.p + LUA_MINSTACK) + ci->top.p = L->top.p + LUA_MINSTACK; L->allowhook = 0; /* cannot call hooks inside a hook */ ci->callstatus |= mask; lua_unlock(L); @@ -332,8 +354,8 @@ void luaD_hook (lua_State *L, int event, int line, lua_lock(L); lua_assert(!L->allowhook); L->allowhook = 1; - ci->top = restorestack(L, ci_top); - L->top = restorestack(L, top); + ci->top.p = restorestack(L, ci_top); + L->top.p = restorestack(L, top); ci->callstatus &= ~mask; } } @@ -364,7 +386,7 @@ void luaD_hookcall (lua_State *L, CallInfo *ci) { */ static void rethook (lua_State *L, CallInfo *ci, int nres) { if (L->hookmask & LUA_MASKRET) { /* is return hook on? */ - StkId firstres = L->top - nres; /* index of first result */ + StkId firstres = L->top.p - nres; /* index of first result */ int delta = 0; /* correction for vararg functions */ int ftransfer; if (isLua(ci)) { @@ -372,10 +394,10 @@ static void rethook (lua_State *L, CallInfo *ci, int nres) { if (p->is_vararg) delta = ci->u.l.nextraargs + p->numparams + 1; } - ci->func += delta; /* if vararg, back to virtual 'func' */ - ftransfer = cast(unsigned short, firstres - ci->func); + ci->func.p += delta; /* if vararg, back to virtual 'func' */ + ftransfer = cast(unsigned short, firstres - ci->func.p); luaD_hook(L, LUA_HOOKRET, -1, ftransfer, nres); /* call it */ - ci->func -= delta; + ci->func.p -= delta; } if (isLua(ci = ci->previous)) L->oldpc = pcRel(ci->u.l.savedpc, ci_func(ci)->p); /* set 'oldpc' */ @@ -394,9 +416,9 @@ StkId luaD_tryfuncTM (lua_State *L, StkId func) { tm = luaT_gettmbyobj(L, s2v(func), TM_CALL); /* (after previous GC) */ if (l_unlikely(ttisnil(tm))) luaG_callerror(L, s2v(func)); /* nothing to call */ - for (p = L->top; p > func; p--) /* open space for metamethod */ + for (p = L->top.p; p > func; p--) /* open space for metamethod */ setobjs2s(L, p, p-1); - L->top++; /* stack space pre-allocated by the caller */ + L->top.p++; /* stack space pre-allocated by the caller */ setobj2s(L, func, tm); /* metamethod is the new function to be called */ return func; } @@ -413,28 +435,29 @@ l_sinline void moveresults (lua_State *L, StkId res, int nres, int wanted) { int i; switch (wanted) { /* handle typical cases separately */ case 0: /* no values needed */ - L->top = res; + L->top.p = res; return; case 1: /* one value needed */ if (nres == 0) /* no results? */ setnilvalue(s2v(res)); /* adjust with nil */ else /* at least one result */ - setobjs2s(L, res, L->top - nres); /* move it to proper place */ - L->top = res + 1; + setobjs2s(L, res, L->top.p - nres); /* move it to proper place */ + L->top.p = res + 1; return; case LUA_MULTRET: wanted = nres; /* we want all results */ break; default: /* two/more results and/or to-be-closed variables */ if (hastocloseCfunc(wanted)) { /* to-be-closed variables? */ - ptrdiff_t savedres = savestack(L, res); L->ci->callstatus |= CIST_CLSRET; /* in case of yields */ L->ci->u2.nres = nres; - luaF_close(L, res, CLOSEKTOP, 1); + res = luaF_close(L, res, CLOSEKTOP, 1); L->ci->callstatus &= ~CIST_CLSRET; - if (L->hookmask) /* if needed, call hook after '__close's */ + if (L->hookmask) { /* if needed, call hook after '__close's */ + ptrdiff_t savedres = savestack(L, res); rethook(L, L->ci, nres); - res = restorestack(L, savedres); /* close and hook can move stack */ + res = restorestack(L, savedres); /* hook can move stack */ + } wanted = decodeNresults(wanted); if (wanted == LUA_MULTRET) wanted = nres; /* we want all results */ @@ -442,14 +465,14 @@ l_sinline void moveresults (lua_State *L, StkId res, int nres, int wanted) { break; } /* generic case */ - firstresult = L->top - nres; /* index of first result */ + firstresult = L->top.p - nres; /* index of first result */ if (nres > wanted) /* extra results? */ nres = wanted; /* don't need them */ for (i = 0; i < nres; i++) /* move all results to correct place */ setobjs2s(L, res + i, firstresult + i); for (; i < wanted; i++) /* complete wanted number of results */ setnilvalue(s2v(res + i)); - L->top = res + wanted; /* top points after the last result */ + L->top.p = res + wanted; /* top points after the last result */ } @@ -464,7 +487,7 @@ void luaD_poscall (lua_State *L, CallInfo *ci, int nres) { if (l_unlikely(L->hookmask && !hastocloseCfunc(wanted))) rethook(L, ci, nres); /* move results to proper place */ - moveresults(L, ci->func, nres, wanted); + moveresults(L, ci->func.p, nres, wanted); /* function cannot be in any of these cases when returning */ lua_assert(!(ci->callstatus & (CIST_HOOKED | CIST_YPCALL | CIST_FIN | CIST_TRAN | CIST_CLSRET))); @@ -479,10 +502,10 @@ void luaD_poscall (lua_State *L, CallInfo *ci, int nres) { l_sinline CallInfo *prepCallInfo (lua_State *L, StkId func, int nret, int mask, StkId top) { CallInfo *ci = L->ci = next_ci(L); /* new frame */ - ci->func = func; + ci->func.p = func; ci->nresults = nret; ci->callstatus = mask; - ci->top = top; + ci->top.p = top; return ci; } @@ -496,10 +519,10 @@ l_sinline int precallC (lua_State *L, StkId func, int nresults, CallInfo *ci; checkstackGCp(L, LUA_MINSTACK, func); /* ensure minimum stack size */ L->ci = ci = prepCallInfo(L, func, nresults, CIST_C, - L->top + LUA_MINSTACK); - lua_assert(ci->top <= L->stack_last); + L->top.p + LUA_MINSTACK); + lua_assert(ci->top.p <= L->stack_last.p); if (l_unlikely(L->hookmask & LUA_MASKCALL)) { - int narg = cast_int(L->top - func) - 1; + int narg = cast_int(L->top.p - func) - 1; luaD_hook(L, LUA_HOOKCALL, -1, 1, narg); } lua_unlock(L); @@ -531,17 +554,17 @@ int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, int nfixparams = p->numparams; int i; checkstackGCp(L, fsize - delta, func); - ci->func -= delta; /* restore 'func' (if vararg) */ + ci->func.p -= delta; /* restore 'func' (if vararg) */ for (i = 0; i < narg1; i++) /* move down function and arguments */ - setobjs2s(L, ci->func + i, func + i); - func = ci->func; /* moved-down function */ + setobjs2s(L, ci->func.p + i, func + i); + func = ci->func.p; /* moved-down function */ for (; narg1 <= nfixparams; narg1++) setnilvalue(s2v(func + narg1)); /* complete missing arguments */ - ci->top = func + 1 + fsize; /* top for new function */ - lua_assert(ci->top <= L->stack_last); + ci->top.p = func + 1 + fsize; /* top for new function */ + lua_assert(ci->top.p <= L->stack_last.p); ci->u.l.savedpc = p->code; /* starting point */ ci->callstatus |= CIST_TAIL; - L->top = func + narg1; /* set top */ + L->top.p = func + narg1; /* set top */ return -1; } default: { /* not a function */ @@ -574,15 +597,15 @@ CallInfo *luaD_precall (lua_State *L, StkId func, int nresults) { case LUA_VLCL: { /* Lua function */ CallInfo *ci; Proto *p = clLvalue(s2v(func))->p; - int narg = cast_int(L->top - func) - 1; /* number of real arguments */ + int narg = cast_int(L->top.p - func) - 1; /* number of real arguments */ int nfixparams = p->numparams; int fsize = p->maxstacksize; /* frame size */ checkstackGCp(L, fsize, func); L->ci = ci = prepCallInfo(L, func, nresults, 0, func + 1 + fsize); ci->u.l.savedpc = p->code; /* starting point */ for (; narg < nfixparams; narg++) - setnilvalue(s2v(L->top++)); /* complete missing arguments */ - lua_assert(ci->top <= L->stack_last); + setnilvalue(s2v(L->top.p++)); /* complete missing arguments */ + lua_assert(ci->top.p <= L->stack_last.p); return ci; } default: { /* not a function */ @@ -598,12 +621,17 @@ CallInfo *luaD_precall (lua_State *L, StkId func, int nresults) { ** Call a function (C or Lua) through C. 'inc' can be 1 (increment ** number of recursive invocations in the C stack) or nyci (the same ** plus increment number of non-yieldable calls). +** This function can be called with some use of EXTRA_STACK, so it should +** check the stack before doing anything else. 'luaD_precall' already +** does that. */ -l_sinline void ccall (lua_State *L, StkId func, int nResults, int inc) { +l_sinline void ccall (lua_State *L, StkId func, int nResults, l_uint32 inc) { CallInfo *ci; L->nCcalls += inc; - if (l_unlikely(getCcalls(L) >= LUAI_MAXCCALLS)) + if (l_unlikely(getCcalls(L) >= LUAI_MAXCCALLS)) { + checkstackp(L, 0, func); /* free any use of EXTRA_STACK */ luaE_checkcstack(L); + } if ((ci = luaD_precall(L, func, nResults)) != NULL) { /* Lua function? */ ci->callstatus = CIST_FRESH; /* mark that it is a "fresh" execute */ luaV_execute(L, ci); /* call it */ @@ -651,8 +679,7 @@ static int finishpcallk (lua_State *L, CallInfo *ci) { else { /* error */ StkId func = restorestack(L, ci->u2.funcidx); L->allowhook = getoah(ci->callstatus); /* restore 'allowhook' */ - luaF_close(L, func, status, 1); /* can yield or raise an error */ - func = restorestack(L, ci->u2.funcidx); /* stack may be moved */ + func = luaF_close(L, func, status, 1); /* can yield or raise an error */ luaD_seterrorobj(L, status, func); luaD_shrinkstack(L); /* restore stack size in case of overflow */ setcistrecst(ci, LUA_OK); /* clear original status */ @@ -740,8 +767,8 @@ static CallInfo *findpcall (lua_State *L) { ** coroutine error handler and should not kill the coroutine.) */ static int resume_error (lua_State *L, const char *msg, int narg) { - L->top -= narg; /* remove args from the stack */ - setsvalue2s(L, L->top, luaS_new(L, msg)); /* push error message */ + L->top.p -= narg; /* remove args from the stack */ + setsvalue2s(L, L->top.p, luaS_new(L, msg)); /* push error message */ api_incr_top(L); lua_unlock(L); return LUA_ERRRUN; @@ -757,7 +784,7 @@ static int resume_error (lua_State *L, const char *msg, int narg) { */ static void resume (lua_State *L, void *ud) { int n = *(cast(int*, ud)); /* number of arguments */ - StkId firstArg = L->top - n; /* first argument */ + StkId firstArg = L->top.p - n; /* first argument */ CallInfo *ci = L->ci; if (L->status == LUA_OK) /* starting a coroutine? */ ccall(L, firstArg - 1, LUA_MULTRET, 0); /* just call its body */ @@ -765,7 +792,7 @@ static void resume (lua_State *L, void *ud) { lua_assert(L->status == LUA_YIELD); L->status = LUA_OK; /* mark that it is running (again) */ if (isLua(ci)) { /* yielded inside a hook? */ - L->top = firstArg; /* discard arguments */ + L->top.p = firstArg; /* discard arguments */ luaV_execute(L, ci); /* just continue running Lua code */ } else { /* 'common' yield */ @@ -808,7 +835,7 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs, if (L->status == LUA_OK) { /* may be starting a coroutine */ if (L->ci != &L->base_ci) /* not in base level? */ return resume_error(L, "cannot resume non-suspended coroutine", nargs); - else if (L->top - (L->ci->func + 1) == nargs) /* no function? */ + else if (L->top.p - (L->ci->func.p + 1) == nargs) /* no function? */ return resume_error(L, "cannot resume dead coroutine", nargs); } else if (L->status != LUA_YIELD) /* ended with errors? */ @@ -826,11 +853,11 @@ LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs, lua_assert(status == L->status); /* normal end or yield */ else { /* unrecoverable error */ L->status = cast_byte(status); /* mark thread as 'dead' */ - luaD_seterrorobj(L, status, L->top); /* push error message */ - L->ci->top = L->top; + luaD_seterrorobj(L, status, L->top.p); /* push error message */ + L->ci->top.p = L->top.p; } *nresults = (status == LUA_YIELD) ? L->ci->u2.nyield - : cast_int(L->top - (L->ci->func + 1)); + : cast_int(L->top.p - (L->ci->func.p + 1)); lua_unlock(L); return status; } @@ -985,7 +1012,7 @@ int luaD_protectedparser (lua_State *L, ZIO *z, const char *name, p.dyd.gt.arr = NULL; p.dyd.gt.size = 0; p.dyd.label.arr = NULL; p.dyd.label.size = 0; luaZ_initbuffer(L, &p.buff); - status = luaD_pcall(L, f_parser, &p, savestack(L, L->top), L->errfunc); + status = luaD_pcall(L, f_parser, &p, savestack(L, L->top.p), L->errfunc); luaZ_freebuffer(L, &p.buff); luaM_freearray(L, p.dyd.actvar.arr, p.dyd.actvar.size); luaM_freearray(L, p.dyd.gt.arr, p.dyd.gt.size); diff --git a/src/lua/ldo.h b/src/lua/ldo.h index 911e67f66..1aa446ad0 100644 --- a/src/lua/ldo.h +++ b/src/lua/ldo.h @@ -8,6 +8,7 @@ #define ldo_h +#include "llimits.h" #include "lobject.h" #include "lstate.h" #include "lzio.h" @@ -23,7 +24,7 @@ ** at every check. */ #define luaD_checkstackaux(L,n,pre,pos) \ - if (l_unlikely(L->stack_last - L->top <= (n))) \ + if (l_unlikely(L->stack_last.p - L->top.p <= (n))) \ { pre; luaD_growstack(L, n, 1); pos; } \ else { condmovestack(L,pre,pos); } @@ -32,11 +33,18 @@ -#define savestack(L,p) ((char *)(p) - (char *)L->stack) -#define restorestack(L,n) ((StkId)((char *)L->stack + (n))) +#define savestack(L,pt) (cast_charp(pt) - cast_charp(L->stack.p)) +#define restorestack(L,n) cast(StkId, cast_charp(L->stack.p) + (n)) /* macro to check stack size, preserving 'p' */ +#define checkstackp(L,n,p) \ + luaD_checkstackaux(L, n, \ + ptrdiff_t t__ = savestack(L, p), /* save 'p' */ \ + p = restorestack(L, t__)) /* 'pos' part: restore 'p' */ + + +/* macro to check stack size and GC, preserving 'p' */ #define checkstackGCp(L,n,p) \ luaD_checkstackaux(L, n, \ ptrdiff_t t__ = savestack(L, p); /* save 'p' */ \ @@ -58,7 +66,8 @@ LUAI_FUNC int luaD_protectedparser (lua_State *L, ZIO *z, const char *name, LUAI_FUNC void luaD_hook (lua_State *L, int event, int line, int fTransfer, int nTransfer); LUAI_FUNC void luaD_hookcall (lua_State *L, CallInfo *ci); -LUAI_FUNC int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, int narg1, int delta); +LUAI_FUNC int luaD_pretailcall (lua_State *L, CallInfo *ci, StkId func, + int narg1, int delta); LUAI_FUNC CallInfo *luaD_precall (lua_State *L, StkId func, int nResults); LUAI_FUNC void luaD_call (lua_State *L, StkId func, int nResults); LUAI_FUNC void luaD_callnoyield (lua_State *L, StkId func, int nResults); diff --git a/src/lua/ldump.c b/src/lua/ldump.c index f848b669c..f231691b7 100644 --- a/src/lua/ldump.c +++ b/src/lua/ldump.c @@ -10,6 +10,7 @@ #include "lprefix.h" +#include #include #include "lua.h" @@ -55,8 +56,11 @@ static void dumpByte (DumpState *D, int y) { } -/* dumpInt Buff Size */ -#define DIBS ((sizeof(size_t) * 8 / 7) + 1) +/* +** 'dumpSize' buffer size: each byte can store up to 7 bits. (The "+6" +** rounds up the division.) +*/ +#define DIBS ((sizeof(size_t) * CHAR_BIT + 6) / 7) static void dumpSize (DumpState *D, size_t x) { lu_byte buff[DIBS]; diff --git a/src/lua/lfunc.c b/src/lua/lfunc.c index f5889a21d..0945f241d 100644 --- a/src/lua/lfunc.c +++ b/src/lua/lfunc.c @@ -50,8 +50,8 @@ void luaF_initupvals (lua_State *L, LClosure *cl) { for (i = 0; i < cl->nupvalues; i++) { GCObject *o = luaC_newobj(L, LUA_VUPVAL, sizeof(UpVal)); UpVal *uv = gco2upv(o); - uv->v = &uv->u.value; /* make it closed */ - setnilvalue(uv->v); + uv->v.p = &uv->u.value; /* make it closed */ + setnilvalue(uv->v.p); cl->upvals[i] = uv; luaC_objbarrier(L, cl, uv); } @@ -62,12 +62,11 @@ void luaF_initupvals (lua_State *L, LClosure *cl) { ** Create a new upvalue at the given level, and link it to the list of ** open upvalues of 'L' after entry 'prev'. **/ -static UpVal *newupval (lua_State *L, int tbc, StkId level, UpVal **prev) { +static UpVal *newupval (lua_State *L, StkId level, UpVal **prev) { GCObject *o = luaC_newobj(L, LUA_VUPVAL, sizeof(UpVal)); UpVal *uv = gco2upv(o); UpVal *next = *prev; - uv->v = s2v(level); /* current value lives in the stack */ - uv->tbc = tbc; + uv->v.p = s2v(level); /* current value lives in the stack */ uv->u.open.next = next; /* link it to list of open upvalues */ uv->u.open.previous = prev; if (next) @@ -96,7 +95,7 @@ UpVal *luaF_findupval (lua_State *L, StkId level) { pp = &p->u.open.next; } /* not found: create a new upvalue after 'pp' */ - return newupval(L, 0, level, pp); + return newupval(L, level, pp); } @@ -106,12 +105,12 @@ UpVal *luaF_findupval (lua_State *L, StkId level) { ** (This function assumes EXTRA_STACK.) */ static void callclosemethod (lua_State *L, TValue *obj, TValue *err, int yy) { - StkId top = L->top; + StkId top = L->top.p; const TValue *tm = luaT_gettmbyobj(L, obj, TM_CLOSE); setobj2s(L, top, tm); /* will call metamethod... */ setobj2s(L, top + 1, obj); /* with 'self' as the 1st argument */ setobj2s(L, top + 2, err); /* and error msg. as 2nd argument */ - L->top = top + 3; /* add function and arguments */ + L->top.p = top + 3; /* add function and arguments */ if (yy) luaD_call(L, top, 0); else @@ -126,7 +125,7 @@ static void callclosemethod (lua_State *L, TValue *obj, TValue *err, int yy) { static void checkclosemth (lua_State *L, StkId level) { const TValue *tm = luaT_gettmbyobj(L, s2v(level), TM_CLOSE); if (ttisnil(tm)) { /* no metamethod? */ - int idx = cast_int(level - L->ci->func); /* variable index */ + int idx = cast_int(level - L->ci->func.p); /* variable index */ const char *vname = luaG_findlocal(L, L->ci, idx, NULL); if (vname == NULL) vname = "?"; luaG_runerror(L, "variable '%s' got a non-closable value", vname); @@ -160,23 +159,23 @@ static void prepcallclosemth (lua_State *L, StkId level, int status, int yy) { ** is used.) */ #define MAXDELTA \ - ((256ul << ((sizeof(L->stack->tbclist.delta) - 1) * 8)) - 1) + ((256ul << ((sizeof(L->stack.p->tbclist.delta) - 1) * 8)) - 1) /* ** Insert a variable in the list of to-be-closed variables. */ void luaF_newtbcupval (lua_State *L, StkId level) { - lua_assert(level > L->tbclist); + lua_assert(level > L->tbclist.p); if (l_isfalse(s2v(level))) return; /* false doesn't need to be closed */ checkclosemth(L, level); /* value must have a close method */ - while (cast_uint(level - L->tbclist) > MAXDELTA) { - L->tbclist += MAXDELTA; /* create a dummy node at maximum delta */ - L->tbclist->tbclist.delta = 0; + while (cast_uint(level - L->tbclist.p) > MAXDELTA) { + L->tbclist.p += MAXDELTA; /* create a dummy node at maximum delta */ + L->tbclist.p->tbclist.delta = 0; } - level->tbclist.delta = cast(unsigned short, level - L->tbclist); - L->tbclist = level; + level->tbclist.delta = cast(unsigned short, level - L->tbclist.p); + L->tbclist.p = level; } @@ -196,10 +195,10 @@ void luaF_closeupval (lua_State *L, StkId level) { StkId upl; /* stack index pointed by 'uv' */ while ((uv = L->openupval) != NULL && (upl = uplevel(uv)) >= level) { TValue *slot = &uv->u.value; /* new position for value */ - lua_assert(uplevel(uv) < L->top); + lua_assert(uplevel(uv) < L->top.p); luaF_unlinkupval(uv); /* remove upvalue from 'openupval' list */ - setobj(L, slot, uv->v); /* move value to upvalue slot */ - uv->v = slot; /* now current value lives here */ + setobj(L, slot, uv->v.p); /* move value to upvalue slot */ + uv->v.p = slot; /* now current value lives here */ if (!iswhite(uv)) { /* neither white nor dead? */ nw2black(uv); /* closed upvalues cannot be gray */ luaC_barrier(L, uv, slot); @@ -209,31 +208,32 @@ void luaF_closeupval (lua_State *L, StkId level) { /* -** Remove firt element from the tbclist plus its dummy nodes. +** Remove first element from the tbclist plus its dummy nodes. */ static void poptbclist (lua_State *L) { - StkId tbc = L->tbclist; + StkId tbc = L->tbclist.p; lua_assert(tbc->tbclist.delta > 0); /* first element cannot be dummy */ tbc -= tbc->tbclist.delta; - while (tbc > L->stack && tbc->tbclist.delta == 0) + while (tbc > L->stack.p && tbc->tbclist.delta == 0) tbc -= MAXDELTA; /* remove dummy nodes */ - L->tbclist = tbc; + L->tbclist.p = tbc; } /* ** Close all upvalues and to-be-closed variables up to the given stack -** level. +** level. Return restored 'level'. */ -void luaF_close (lua_State *L, StkId level, int status, int yy) { +StkId luaF_close (lua_State *L, StkId level, int status, int yy) { ptrdiff_t levelrel = savestack(L, level); luaF_closeupval(L, level); /* first, close the upvalues */ - while (L->tbclist >= level) { /* traverse tbc's down to that level */ - StkId tbc = L->tbclist; /* get variable index */ + while (L->tbclist.p >= level) { /* traverse tbc's down to that level */ + StkId tbc = L->tbclist.p; /* get variable index */ poptbclist(L); /* remove it from list */ prepcallclosemth(L, tbc, status, yy); /* close variable */ level = restorestack(L, levelrel); } + return level; } diff --git a/src/lua/lfunc.h b/src/lua/lfunc.h index dc1cebccd..3be265efb 100644 --- a/src/lua/lfunc.h +++ b/src/lua/lfunc.h @@ -29,10 +29,10 @@ #define MAXUPVAL 255 -#define upisopen(up) ((up)->v != &(up)->u.value) +#define upisopen(up) ((up)->v.p != &(up)->u.value) -#define uplevel(up) check_exp(upisopen(up), cast(StkId, (up)->v)) +#define uplevel(up) check_exp(upisopen(up), cast(StkId, (up)->v.p)) /* @@ -54,7 +54,7 @@ LUAI_FUNC void luaF_initupvals (lua_State *L, LClosure *cl); LUAI_FUNC UpVal *luaF_findupval (lua_State *L, StkId level); LUAI_FUNC void luaF_newtbcupval (lua_State *L, StkId level); LUAI_FUNC void luaF_closeupval (lua_State *L, StkId level); -LUAI_FUNC void luaF_close (lua_State *L, StkId level, int status, int yy); +LUAI_FUNC StkId luaF_close (lua_State *L, StkId level, int status, int yy); LUAI_FUNC void luaF_unlinkupval (UpVal *uv); LUAI_FUNC void luaF_freeproto (lua_State *L, Proto *f); LUAI_FUNC const char *luaF_getlocalname (const Proto *func, int local_number, diff --git a/src/lua/lgc.c b/src/lua/lgc.c index 42a73d813..a3094ff57 100644 --- a/src/lua/lgc.c +++ b/src/lua/lgc.c @@ -252,12 +252,13 @@ void luaC_fix (lua_State *L, GCObject *o) { /* -** create a new collectable object (with given type and size) and link -** it to 'allgc' list. +** create a new collectable object (with given type, size, and offset) +** and link it to 'allgc' list. */ -GCObject *luaC_newobj (lua_State *L, int tt, size_t sz) { +GCObject *luaC_newobjdt (lua_State *L, int tt, size_t sz, size_t offset) { global_State *g = G(L); - GCObject *o = cast(GCObject *, luaM_newobject(L, novariant(tt), sz)); + char *p = cast_charp(luaM_newobject(L, novariant(tt), sz)); + GCObject *o = cast(GCObject *, p + offset); o->marked = luaC_white(g); o->tt = tt; o->next = g->allgc; @@ -265,6 +266,11 @@ GCObject *luaC_newobj (lua_State *L, int tt, size_t sz) { return o; } + +GCObject *luaC_newobj (lua_State *L, int tt, size_t sz) { + return luaC_newobjdt(L, tt, sz, 0); +} + /* }====================================================== */ @@ -301,7 +307,7 @@ static void reallymarkobject (global_State *g, GCObject *o) { set2gray(uv); /* open upvalues are kept gray */ else set2black(uv); /* closed upvalues are visited here */ - markvalue(g, uv->v); /* mark its content */ + markvalue(g, uv->v.p); /* mark its content */ break; } case LUA_VUSERDATA: { @@ -376,7 +382,7 @@ static int remarkupvals (global_State *g) { work++; if (!iswhite(uv)) { /* upvalue already visited? */ lua_assert(upisopen(uv) && isgray(uv)); - markvalue(g, uv->v); /* mark its value */ + markvalue(g, uv->v.p); /* mark its value */ } } } @@ -620,19 +626,19 @@ static int traverseLclosure (global_State *g, LClosure *cl) { */ static int traversethread (global_State *g, lua_State *th) { UpVal *uv; - StkId o = th->stack; + StkId o = th->stack.p; if (isold(th) || g->gcstate == GCSpropagate) linkgclist(th, g->grayagain); /* insert into 'grayagain' list */ if (o == NULL) return 1; /* stack not completely built yet */ lua_assert(g->gcstate == GCSatomic || th->openupval == NULL || isintwups(th)); - for (; o < th->top; o++) /* mark live elements in the stack */ + for (; o < th->top.p; o++) /* mark live elements in the stack */ markvalue(g, s2v(o)); for (uv = th->openupval; uv != NULL; uv = uv->u.open.next) markobject(g, uv); /* open upvalues cannot be collected */ if (g->gcstate == GCSatomic) { /* final traversal? */ - for (; o < th->stack_last + EXTRA_STACK; o++) + for (; o < th->stack_last.p + EXTRA_STACK; o++) setnilvalue(s2v(o)); /* clear dead stack slice */ /* 'remarkupvals' may have removed thread from 'twups' list */ if (!isintwups(th) && th->openupval != NULL) { @@ -892,7 +898,7 @@ static GCObject *udata2finalize (global_State *g) { static void dothecall (lua_State *L, void *ud) { UNUSED(ud); - luaD_callnoyield(L, L->top - 2, 0); + luaD_callnoyield(L, L->top.p - 2, 0); } @@ -909,16 +915,16 @@ static void GCTM (lua_State *L) { int oldgcstp = g->gcstp; g->gcstp |= GCSTPGC; /* avoid GC steps */ L->allowhook = 0; /* stop debug hooks during GC metamethod */ - setobj2s(L, L->top++, tm); /* push finalizer... */ - setobj2s(L, L->top++, &v); /* ... and its argument */ + setobj2s(L, L->top.p++, tm); /* push finalizer... */ + setobj2s(L, L->top.p++, &v); /* ... and its argument */ L->ci->callstatus |= CIST_FIN; /* will run a finalizer */ - status = luaD_pcall(L, dothecall, NULL, savestack(L, L->top - 2), 0); + status = luaD_pcall(L, dothecall, NULL, savestack(L, L->top.p - 2), 0); L->ci->callstatus &= ~CIST_FIN; /* not running a finalizer anymore */ L->allowhook = oldah; /* restore hooks */ g->gcstp = oldgcstp; /* restore state */ if (l_unlikely(status != LUA_OK)) { /* error while running __gc? */ luaE_warnerror(L, "__gc"); - L->top--; /* pops error object */ + L->top.p--; /* pops error object */ } } } @@ -1041,7 +1047,25 @@ void luaC_checkfinalizer (lua_State *L, GCObject *o, Table *mt) { ** ======================================================= */ -static void setpause (global_State *g); + +/* +** Set the "time" to wait before starting a new GC cycle; cycle will +** start when memory use hits the threshold of ('estimate' * pause / +** PAUSEADJ). (Division by 'estimate' should be OK: it cannot be zero, +** because Lua cannot even start with less than PAUSEADJ bytes). +*/ +static void setpause (global_State *g) { + l_mem threshold, debt; + int pause = getgcparam(g->gcpause); + l_mem estimate = g->GCestimate / PAUSEADJ; /* adjust 'estimate' */ + lua_assert(estimate > 0); + threshold = (pause < MAX_LMEM / estimate) /* overflow? */ + ? estimate * pause /* no overflow */ + : MAX_LMEM; /* overflow; truncate to maximum */ + debt = gettotalbytes(g) - threshold; + if (debt > 0) debt = 0; + luaE_setdebt(g, debt); +} /* @@ -1285,6 +1309,15 @@ static void atomic2gen (lua_State *L, global_State *g) { } +/* +** Set debt for the next minor collection, which will happen when +** memory grows 'genminormul'%. +*/ +static void setminordebt (global_State *g) { + luaE_setdebt(g, -(cast(l_mem, (gettotalbytes(g) / 100)) * g->genminormul)); +} + + /* ** Enter generational mode. Must go until the end of an atomic cycle ** to ensure that all objects are correctly marked and weak tables @@ -1297,6 +1330,7 @@ static lu_mem entergen (lua_State *L, global_State *g) { luaC_runtilstate(L, bitmask(GCSpropagate)); /* start new cycle */ numobjs = atomic(L); /* propagates all and then do the atomic stuff */ atomic2gen(L, g); + setminordebt(g); /* set debt assuming next cycle will be minor */ return numobjs; } @@ -1342,15 +1376,6 @@ static lu_mem fullgen (lua_State *L, global_State *g) { } -/* -** Set debt for the next minor collection, which will happen when -** memory grows 'genminormul'%. -*/ -static void setminordebt (global_State *g) { - luaE_setdebt(g, -(cast(l_mem, (gettotalbytes(g) / 100)) * g->genminormul)); -} - - /* ** Does a major collection after last collection was a "bad collection". ** @@ -1422,8 +1447,8 @@ static void genstep (lua_State *L, global_State *g) { lu_mem numobjs = fullgen(L, g); /* do a major collection */ if (gettotalbytes(g) < majorbase + (majorinc / 2)) { /* collected at least half of memory growth since last major - collection; keep doing minor collections */ - setminordebt(g); + collection; keep doing minor collections. */ + lua_assert(g->lastatomic == 0); } else { /* bad collection */ g->lastatomic = numobjs; /* signal that last collection was bad */ @@ -1449,26 +1474,6 @@ static void genstep (lua_State *L, global_State *g) { */ -/* -** Set the "time" to wait before starting a new GC cycle; cycle will -** start when memory use hits the threshold of ('estimate' * pause / -** PAUSEADJ). (Division by 'estimate' should be OK: it cannot be zero, -** because Lua cannot even start with less than PAUSEADJ bytes). -*/ -static void setpause (global_State *g) { - l_mem threshold, debt; - int pause = getgcparam(g->gcpause); - l_mem estimate = g->GCestimate / PAUSEADJ; /* adjust 'estimate' */ - lua_assert(estimate > 0); - threshold = (pause < MAX_LMEM / estimate) /* overflow? */ - ? estimate * pause /* no overflow */ - : MAX_LMEM; /* overflow; truncate to maximum */ - debt = gettotalbytes(g) - threshold; - if (debt > 0) debt = 0; - luaE_setdebt(g, debt); -} - - /* ** Enter first sweep phase. ** The call to 'sweeptolive' makes the pointer point to an object @@ -1676,12 +1681,15 @@ static void incstep (lua_State *L, global_State *g) { } /* -** performs a basic GC step if collector is running +** Performs a basic GC step if collector is running. (If collector is +** not running, set a reasonable debt to avoid it being called at +** every single check.) */ void luaC_step (lua_State *L) { global_State *g = G(L); - lua_assert(!g->gcemergency); - if (gcrunning(g)) { /* running? */ + if (!gcrunning(g)) /* not running? */ + luaE_setdebt(g, -2000); + else { if(isdecGCmodegen(g)) genstep(L, g); else diff --git a/src/lua/lgc.h b/src/lua/lgc.h index 4a125634b..538f6edcc 100644 --- a/src/lua/lgc.h +++ b/src/lua/lgc.h @@ -172,24 +172,27 @@ #define luaC_checkGC(L) luaC_condGC(L,(void)0,(void)0) -#define luaC_barrier(L,p,v) ( \ - (iscollectable(v) && isblack(p) && iswhite(gcvalue(v))) ? \ - luaC_barrier_(L,obj2gco(p),gcvalue(v)) : cast_void(0)) - -#define luaC_barrierback(L,p,v) ( \ - (iscollectable(v) && isblack(p) && iswhite(gcvalue(v))) ? \ - luaC_barrierback_(L,p) : cast_void(0)) - #define luaC_objbarrier(L,p,o) ( \ (isblack(p) && iswhite(o)) ? \ luaC_barrier_(L,obj2gco(p),obj2gco(o)) : cast_void(0)) +#define luaC_barrier(L,p,v) ( \ + iscollectable(v) ? luaC_objbarrier(L,p,gcvalue(v)) : cast_void(0)) + +#define luaC_objbarrierback(L,p,o) ( \ + (isblack(p) && iswhite(o)) ? luaC_barrierback_(L,p) : cast_void(0)) + +#define luaC_barrierback(L,p,v) ( \ + iscollectable(v) ? luaC_objbarrierback(L, p, gcvalue(v)) : cast_void(0)) + LUAI_FUNC void luaC_fix (lua_State *L, GCObject *o); LUAI_FUNC void luaC_freeallobjects (lua_State *L); LUAI_FUNC void luaC_step (lua_State *L); LUAI_FUNC void luaC_runtilstate (lua_State *L, int statesmask); LUAI_FUNC void luaC_fullgc (lua_State *L, int isemergency); LUAI_FUNC GCObject *luaC_newobj (lua_State *L, int tt, size_t sz); +LUAI_FUNC GCObject *luaC_newobjdt (lua_State *L, int tt, size_t sz, + size_t offset); LUAI_FUNC void luaC_barrier_ (lua_State *L, GCObject *o, GCObject *v); LUAI_FUNC void luaC_barrierback_ (lua_State *L, GCObject *o); LUAI_FUNC void luaC_checkfinalizer (lua_State *L, GCObject *o, Table *mt); diff --git a/src/lua/linit.c b/src/lua/linit.c index 9a5bcfdcc..69808f84f 100644 --- a/src/lua/linit.c +++ b/src/lua/linit.c @@ -50,9 +50,6 @@ static const luaL_Reg loadedlibs[] = { {LUA_MATHLIBNAME, luaopen_math}, {LUA_UTF8LIBNAME, luaopen_utf8}, {LUA_DBLIBNAME, luaopen_debug}, - /****** Pi-hole modification ******/ - {LUA_PIHOLELIBNAME, luaopen_pihole}, - /**********************************/ {NULL, NULL} }; diff --git a/src/lua/llex.c b/src/lua/llex.c index e99151787..5fc39a5cd 100644 --- a/src/lua/llex.c +++ b/src/lua/llex.c @@ -128,7 +128,7 @@ l_noret luaX_syntaxerror (LexState *ls, const char *msg) { ** ensuring there is only one copy of each unique string. The table ** here is used as a set: the string enters as the key, while its value ** is irrelevant. We use the string itself as the value only because it -** is a TValue readly available. Later, the code generation can change +** is a TValue readily available. Later, the code generation can change ** this value. */ TString *luaX_newstring (LexState *ls, const char *str, size_t l) { @@ -138,12 +138,12 @@ TString *luaX_newstring (LexState *ls, const char *str, size_t l) { if (!ttisnil(o)) /* string already present? */ ts = keystrval(nodefromval(o)); /* get saved copy */ else { /* not in use yet */ - TValue *stv = s2v(L->top++); /* reserve stack space for string */ + TValue *stv = s2v(L->top.p++); /* reserve stack space for string */ setsvalue(L, stv, ts); /* temporarily anchor the string */ luaH_finishset(L, ls->h, stv, o, stv); /* t[string] = string */ /* table is not a metatable, so it does not need to invalidate cache */ luaC_checkGC(L); - L->top--; /* remove string from stack */ + L->top.p--; /* remove string from stack */ } return ts; } diff --git a/src/lua/llimits.h b/src/lua/llimits.h index 52a32f92e..1c826f7be 100644 --- a/src/lua/llimits.h +++ b/src/lua/llimits.h @@ -71,11 +71,24 @@ typedef signed char ls_byte; /* -** conversion of pointer to unsigned integer: -** this is for hashing only; there is no problem if the integer -** cannot hold the whole pointer value +** conversion of pointer to unsigned integer: this is for hashing only; +** there is no problem if the integer cannot hold the whole pointer +** value. (In strict ISO C this may cause undefined behavior, but no +** actual machine seems to bother.) */ -#define point2uint(p) ((unsigned int)((size_t)(p) & UINT_MAX)) +#if !defined(LUA_USE_C89) && defined(__STDC_VERSION__) && \ + __STDC_VERSION__ >= 199901L +#include +#if defined(UINTPTR_MAX) /* even in C99 this type is optional */ +#define L_P2I uintptr_t +#else /* no 'intptr'? */ +#define L_P2I uintmax_t /* use the largest available integer */ +#endif +#else /* C89 option */ +#define L_P2I size_t +#endif + +#define point2uint(p) ((unsigned int)((L_P2I)(p) & UINT_MAX)) diff --git a/src/lua/lmathlib.c b/src/lua/lmathlib.c index e0c61a168..d0b1e1e5d 100644 --- a/src/lua/lmathlib.c +++ b/src/lua/lmathlib.c @@ -267,7 +267,7 @@ static int math_type (lua_State *L) { /* try to find an integer type with at least 64 bits */ -#if (ULONG_MAX >> 31 >> 31) >= 3 +#if ((ULONG_MAX >> 31) >> 31) >= 3 /* 'long' has at least 64 bits */ #define Rand64 unsigned long @@ -277,9 +277,9 @@ static int math_type (lua_State *L) { /* there is a 'long long' type (which must have at least 64 bits) */ #define Rand64 unsigned long long -#elif (LUA_MAXUNSIGNED >> 31 >> 31) >= 3 +#elif ((LUA_MAXUNSIGNED >> 31) >> 31) >= 3 -/* 'lua_Integer' has at least 64 bits */ +/* 'lua_Unsigned' has at least 64 bits */ #define Rand64 lua_Unsigned #endif @@ -500,12 +500,12 @@ static lua_Number I2d (Rand64 x) { /* convert a 'Rand64' to a 'lua_Unsigned' */ static lua_Unsigned I2UInt (Rand64 x) { - return ((lua_Unsigned)trim32(x.h) << 31 << 1) | (lua_Unsigned)trim32(x.l); + return (((lua_Unsigned)trim32(x.h) << 31) << 1) | (lua_Unsigned)trim32(x.l); } /* convert a 'lua_Unsigned' to a 'Rand64' */ static Rand64 Int2I (lua_Unsigned n) { - return packI((lu_int32)(n >> 31 >> 1), (lu_int32)n); + return packI((lu_int32)((n >> 31) >> 1), (lu_int32)n); } #endif /* } */ diff --git a/src/lua/lmem.c b/src/lua/lmem.c index 9029d588c..9800a86fc 100644 --- a/src/lua/lmem.c +++ b/src/lua/lmem.c @@ -22,25 +22,6 @@ #include "lstate.h" -#if defined(EMERGENCYGCTESTS) -/* -** First allocation will fail whenever not building initial state. -** (This fail will trigger 'tryagain' and a full GC cycle at every -** allocation.) -*/ -static void *firsttry (global_State *g, void *block, size_t os, size_t ns) { - if (completestate(g) && ns > 0) /* frees never fail */ - return NULL; /* fail */ - else /* normal allocation */ - return (*g->frealloc)(g->ud, block, os, ns); -} -#else -#define firsttry(g,block,os,ns) ((*g->frealloc)(g->ud, block, os, ns)) -#endif - - - - /* ** About the realloc function: @@ -60,6 +41,43 @@ static void *firsttry (global_State *g, void *block, size_t os, size_t ns) { */ +/* +** Macro to call the allocation function. +*/ +#define callfrealloc(g,block,os,ns) ((*g->frealloc)(g->ud, block, os, ns)) + + +/* +** When an allocation fails, it will try again after an emergency +** collection, except when it cannot run a collection. The GC should +** not be called while the state is not fully built, as the collector +** is not yet fully initialized. Also, it should not be called when +** 'gcstopem' is true, because then the interpreter is in the middle of +** a collection step. +*/ +#define cantryagain(g) (completestate(g) && !g->gcstopem) + + + + +#if defined(EMERGENCYGCTESTS) +/* +** First allocation will fail except when freeing a block (frees never +** fail) and when it cannot try again; this fail will trigger 'tryagain' +** and a full GC cycle at every allocation. +*/ +static void *firsttry (global_State *g, void *block, size_t os, size_t ns) { + if (ns > 0 && cantryagain(g)) + return NULL; /* fail */ + else /* normal allocation */ + return callfrealloc(g, block, os, ns); +} +#else +#define firsttry(g,block,os,ns) callfrealloc(g, block, os, ns) +#endif + + + /* @@ -132,7 +150,7 @@ l_noret luaM_toobig (lua_State *L) { void luaM_free_ (lua_State *L, void *block, size_t osize) { global_State *g = G(L); lua_assert((osize == 0) == (block == NULL)); - (*g->frealloc)(g->ud, block, osize, 0); + callfrealloc(g, block, osize, 0); g->GCdebt -= osize; } @@ -140,19 +158,15 @@ void luaM_free_ (lua_State *L, void *block, size_t osize) { /* ** In case of allocation fail, this function will do an emergency ** collection to free some memory and then try the allocation again. -** The GC should not be called while state is not fully built, as the -** collector is not yet fully initialized. Also, it should not be called -** when 'gcstopem' is true, because then the interpreter is in the -** middle of a collection step. */ static void *tryagain (lua_State *L, void *block, size_t osize, size_t nsize) { global_State *g = G(L); - if (completestate(g) && !g->gcstopem) { + if (cantryagain(g)) { luaC_fullgc(L, 1); /* try to free some memory... */ - return (*g->frealloc)(g->ud, block, osize, nsize); /* try again */ + return callfrealloc(g, block, osize, nsize); /* try again */ } - else return NULL; /* cannot free any memory without a full state */ + else return NULL; /* cannot run an emergency collection */ } diff --git a/src/lua/loadlib.c b/src/lua/loadlib.c index 6f9fa3736..d792dffaa 100644 --- a/src/lua/loadlib.c +++ b/src/lua/loadlib.c @@ -708,8 +708,13 @@ static const luaL_Reg ll_funcs[] = { static void createsearcherstable (lua_State *L) { - static const lua_CFunction searchers[] = - {searcher_preload, searcher_Lua, searcher_C, searcher_Croot, NULL}; + static const lua_CFunction searchers[] = { + searcher_preload, + searcher_Lua, + searcher_C, + searcher_Croot, + NULL + }; int i; /* create 'searchers' table */ lua_createtable(L, sizeof(searchers)/sizeof(searchers[0]) - 1, 0); diff --git a/src/lua/lobject.c b/src/lua/lobject.c index 301aa900b..f73ffc6d9 100644 --- a/src/lua/lobject.c +++ b/src/lua/lobject.c @@ -62,7 +62,7 @@ static lua_Integer intarith (lua_State *L, int op, lua_Integer v1, case LUA_OPBOR: return intop(|, v1, v2); case LUA_OPBXOR: return intop(^, v1, v2); case LUA_OPSHL: return luaV_shiftl(v1, v2); - case LUA_OPSHR: return luaV_shiftl(v1, -v2); + case LUA_OPSHR: return luaV_shiftr(v1, v2); case LUA_OPUNM: return intop(-, 0, v1); case LUA_OPBNOT: return intop(^, ~l_castS2U(0), v1); default: lua_assert(0); return 0; @@ -386,29 +386,39 @@ void luaO_tostring (lua_State *L, TValue *obj) { ** =================================================================== */ -/* size for buffer space used by 'luaO_pushvfstring' */ -#define BUFVFS 200 +/* +** Size for buffer space used by 'luaO_pushvfstring'. It should be +** (LUA_IDSIZE + MAXNUMBER2STR) + a minimal space for basic messages, +** so that 'luaG_addinfo' can work directly on the buffer. +*/ +#define BUFVFS (LUA_IDSIZE + MAXNUMBER2STR + 95) /* buffer used by 'luaO_pushvfstring' */ typedef struct BuffFS { lua_State *L; - int pushed; /* number of string pieces already on the stack */ + int pushed; /* true if there is a part of the result on the stack */ int blen; /* length of partial string in 'space' */ char space[BUFVFS]; /* holds last part of the result */ } BuffFS; /* -** Push given string to the stack, as part of the buffer, and -** join the partial strings in the stack into one. +** Push given string to the stack, as part of the result, and +** join it to previous partial result if there is one. +** It may call 'luaV_concat' while using one slot from EXTRA_STACK. +** This call cannot invoke metamethods, as both operands must be +** strings. It can, however, raise an error if the result is too +** long. In that case, 'luaV_concat' frees the extra slot before +** raising the error. */ -static void pushstr (BuffFS *buff, const char *str, size_t l) { +static void pushstr (BuffFS *buff, const char *str, size_t lstr) { lua_State *L = buff->L; - setsvalue2s(L, L->top, luaS_newlstr(L, str, l)); - L->top++; /* may use one extra slot */ - buff->pushed++; - luaV_concat(L, buff->pushed); /* join partial results into one */ - buff->pushed = 1; + setsvalue2s(L, L->top.p, luaS_newlstr(L, str, lstr)); + L->top.p++; /* may use one slot from EXTRA_STACK */ + if (!buff->pushed) /* no previous string on the stack? */ + buff->pushed = 1; /* now there is one */ + else /* join previous string with new one */ + luaV_concat(L, 2); } @@ -454,7 +464,7 @@ static void addstr2buff (BuffFS *buff, const char *str, size_t slen) { /* -** Add a number to the buffer. +** Add a numeral to the buffer. */ static void addnum2buff (BuffFS *buff, TValue *num) { char *numbuff = getbuff(buff, MAXNUMBER2STR); @@ -532,7 +542,7 @@ const char *luaO_pushvfstring (lua_State *L, const char *fmt, va_list argp) { addstr2buff(&buff, fmt, strlen(fmt)); /* rest of 'fmt' */ clearbuff(&buff); /* empty buffer into the stack */ lua_assert(buff.pushed == 1); - return svalue(s2v(L->top - 1)); + return svalue(s2v(L->top.p - 1)); } diff --git a/src/lua/lobject.h b/src/lua/lobject.h index 0e05b3e42..556608e4a 100644 --- a/src/lua/lobject.h +++ b/src/lua/lobject.h @@ -52,6 +52,8 @@ typedef union Value { lua_CFunction f; /* light C functions */ lua_Integer i; /* integer numbers */ lua_Number n; /* float numbers */ + /* not used, but may avoid warnings for uninitialized value */ + lu_byte ub; } Value; @@ -155,6 +157,17 @@ typedef union StackValue { /* index to stack elements */ typedef StackValue *StkId; + +/* +** When reallocating the stack, change all pointers to the stack into +** proper offsets. +*/ +typedef union { + StkId p; /* actual pointer */ + ptrdiff_t offset; /* used while the stack is being reallocated */ +} StkIdRel; + + /* convert a 'StackValue' to a 'TValue' */ #define s2v(o) (&(o)->val) @@ -615,8 +628,10 @@ typedef struct Proto { */ typedef struct UpVal { CommonHeader; - lu_byte tbc; /* true if it represents a to-be-closed variable */ - TValue *v; /* points to stack or to its own value */ + union { + TValue *p; /* points to stack or to its own value */ + ptrdiff_t offset; /* used while the stack is being reallocated */ + } v; union { struct { /* (when open) */ struct UpVal *next; /* linked list */ diff --git a/src/lua/lopcodes.h b/src/lua/lopcodes.h index 7c2745159..4c5514539 100644 --- a/src/lua/lopcodes.h +++ b/src/lua/lopcodes.h @@ -21,7 +21,7 @@ iABC C(8) | B(8) |k| A(8) | Op(7) | iABx Bx(17) | A(8) | Op(7) | iAsBx sBx (signed)(17) | A(8) | Op(7) | iAx Ax(25) | Op(7) | -isJ sJ(25) | Op(7) | +isJ sJ (signed)(25) | Op(7) | A signed argument is represented in excess K: the represented value is the written unsigned value minus K, where K is half the maximum for the diff --git a/src/lua/loslib.c b/src/lua/loslib.c index 3e20d622b..ad5a92768 100644 --- a/src/lua/loslib.c +++ b/src/lua/loslib.c @@ -30,23 +30,14 @@ */ #if !defined(LUA_STRFTIMEOPTIONS) /* { */ -/* options for ANSI C 89 (only 1-char options) */ -#define L_STRFTIMEC89 "aAbBcdHIjmMpSUwWxXyYZ%" - -/* options for ISO C 99 and POSIX */ -#define L_STRFTIMEC99 "aAbBcCdDeFgGhHIjmMnprRStTuUVwWxXyYzZ%" \ - "||" "EcECExEXEyEY" "OdOeOHOIOmOMOSOuOUOVOwOWOy" /* two-char options */ - -/* options for Windows */ -#define L_STRFTIMEWIN "aAbBcdHIjmMpSUwWxXyYzZ%" \ - "||" "#c#x#d#H#I#j#m#M#S#U#w#W#y#Y" /* two-char options */ - #if defined(LUA_USE_WINDOWS) -#define LUA_STRFTIMEOPTIONS L_STRFTIMEWIN -#elif defined(LUA_USE_C89) -#define LUA_STRFTIMEOPTIONS L_STRFTIMEC89 +#define LUA_STRFTIMEOPTIONS "aAbBcdHIjmMpSUwWxXyYzZ%" \ + "||" "#c#x#d#H#I#j#m#M#S#U#w#W#y#Y" /* two-char options */ +#elif defined(LUA_USE_C89) /* ANSI C 89 (only 1-char options) */ +#define LUA_STRFTIMEOPTIONS "aAbBcdHIjmMpSUwWxXyYZ%" #else /* C99 specification */ -#define LUA_STRFTIMEOPTIONS L_STRFTIMEC99 +#define LUA_STRFTIMEOPTIONS "aAbBcCdDeFgGhHIjmMnprRStTuUVwWxXyYzZ%" \ + "||" "EcECExEXEyEY" "OdOeOHOIOmOMOSOuOUOVOwOWOy" /* two-char options */ #endif #endif /* } */ @@ -138,12 +129,21 @@ /* }================================================================== */ +#if !defined(l_system) +#if defined(LUA_USE_IOS) +/* Despite claiming to be ISO C, iOS does not implement 'system'. */ +#define l_system(cmd) ((cmd) == NULL ? 0 : -1) +#else +#define l_system(cmd) system(cmd) /* default definition */ +#endif +#endif + static int os_execute (lua_State *L) { const char *cmd = luaL_optstring(L, 1, NULL); int stat; errno = 0; - stat = system(cmd); + stat = l_system(cmd); if (cmd != NULL) return luaL_execresult(L, stat); else { @@ -260,9 +260,7 @@ static int getfield (lua_State *L, const char *key, int d, int delta) { res = d; } else { - /* unsigned avoids overflow when lua_Integer has 32 bits */ - if (!(res >= 0 ? (lua_Unsigned)res <= (lua_Unsigned)INT_MAX + delta - : (lua_Integer)INT_MIN + delta <= res)) + if (!(res >= 0 ? res - delta <= INT_MAX : INT_MIN + delta <= res)) return luaL_error(L, "field '%s' is out-of-bound", key); res -= delta; } diff --git a/src/lua/lparser.c b/src/lua/lparser.c index 3abe3d751..b745f236f 100644 --- a/src/lua/lparser.c +++ b/src/lua/lparser.c @@ -468,6 +468,7 @@ static void singlevar (LexState *ls, expdesc *var) { expdesc key; singlevaraux(fs, ls->envn, var, 1); /* get environment variable */ lua_assert(var->k != VVOID); /* this one must exist */ + luaK_exp2anyregup(fs, var); /* but could be a constant */ codestring(&key, varname); /* key is variable name */ luaK_indexed(fs, var, &key); /* env[varname] */ } @@ -520,12 +521,12 @@ static l_noret jumpscopeerror (LexState *ls, Labeldesc *gt) { /* ** Solves the goto at index 'g' to given 'label' and removes it -** from the list of pending goto's. +** from the list of pending gotos. ** If it jumps into the scope of some variable, raises an error. */ static void solvegoto (LexState *ls, int g, Labeldesc *label) { int i; - Labellist *gl = &ls->dyd->gt; /* list of goto's */ + Labellist *gl = &ls->dyd->gt; /* list of gotos */ Labeldesc *gt = &gl->arr[g]; /* goto to be resolved */ lua_assert(eqstr(gt->name, label->name)); if (l_unlikely(gt->nactvar < label->nactvar)) /* enter some scope? */ @@ -579,7 +580,7 @@ static int newgotoentry (LexState *ls, TString *name, int line, int pc) { /* ** Solves forward jumps. Check whether new label 'lb' matches any ** pending gotos in current block and solves them. Return true -** if any of the goto's need to close upvalues. +** if any of the gotos need to close upvalues. */ static int solvegotos (LexState *ls, Labeldesc *lb) { Labellist *gl = &ls->dyd->gt; @@ -600,7 +601,7 @@ static int solvegotos (LexState *ls, Labeldesc *lb) { /* ** Create a new label with the given 'name' at the given 'line'. ** 'last' tells whether label is the last non-op statement in its -** block. Solves all pending goto's to this new label and adds +** block. Solves all pending gotos to this new label and adds ** a close instruction if necessary. ** Returns true iff it added a close instruction. */ @@ -673,19 +674,19 @@ static void leaveblock (FuncState *fs) { LexState *ls = fs->ls; int hasclose = 0; int stklevel = reglevel(fs, bl->nactvar); /* level outside the block */ - if (bl->isloop) /* fix pending breaks? */ + removevars(fs, bl->nactvar); /* remove block locals */ + lua_assert(bl->nactvar == fs->nactvar); /* back to level on entry */ + if (bl->isloop) /* has to fix pending breaks? */ hasclose = createlabel(ls, luaS_newliteral(ls->L, "break"), 0, 0); - if (!hasclose && bl->previous && bl->upval) + if (!hasclose && bl->previous && bl->upval) /* still need a 'close'? */ luaK_codeABC(fs, OP_CLOSE, stklevel, 0, 0); - fs->bl = bl->previous; - removevars(fs, bl->nactvar); - lua_assert(bl->nactvar == fs->nactvar); fs->freereg = stklevel; /* free registers */ ls->dyd->label.n = bl->firstlabel; /* remove local labels */ - if (bl->previous) /* inner block? */ - movegotosout(fs, bl); /* update pending gotos to outer block */ + fs->bl = bl->previous; /* current block now is previous one */ + if (bl->previous) /* was it a nested block? */ + movegotosout(fs, bl); /* update pending gotos to enclosing block */ else { - if (bl->firstgoto < ls->dyd->gt.n) /* pending gotos in outer block? */ + if (bl->firstgoto < ls->dyd->gt.n) /* still pending gotos? */ undefgoto(ls, &ls->dyd->gt.arr[bl->firstgoto]); /* error */ } } @@ -1943,10 +1944,10 @@ LClosure *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, LexState lexstate; FuncState funcstate; LClosure *cl = luaF_newLclosure(L, 1); /* create main closure */ - setclLvalue2s(L, L->top, cl); /* anchor it (to avoid being collected) */ + setclLvalue2s(L, L->top.p, cl); /* anchor it (to avoid being collected) */ luaD_inctop(L); lexstate.h = luaH_new(L); /* create table for scanner */ - sethvalue2s(L, L->top, lexstate.h); /* anchor it */ + sethvalue2s(L, L->top.p, lexstate.h); /* anchor it */ luaD_inctop(L); funcstate.f = cl->p = luaF_newproto(L); luaC_objbarrier(L, cl, cl->p); @@ -1960,7 +1961,7 @@ LClosure *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, lua_assert(!funcstate.prev && funcstate.nups == 1 && !lexstate.fs); /* all scopes should be correctly finished */ lua_assert(dyd->actvar.n == 0 && dyd->gt.n == 0 && dyd->label.n == 0); - L->top--; /* remove scanner's table */ + L->top.p--; /* remove scanner's table */ return cl; /* closure is on the stack, too */ } diff --git a/src/lua/lstate.c b/src/lua/lstate.c index 1ffe1a0f7..1fbefb4b1 100644 --- a/src/lua/lstate.c +++ b/src/lua/lstate.c @@ -180,33 +180,33 @@ LUAI_FUNC void luaE_incCstack (lua_State *L) { static void stack_init (lua_State *L1, lua_State *L) { int i; CallInfo *ci; /* initialize stack array */ - L1->stack = luaM_newvector(L, BASIC_STACK_SIZE + EXTRA_STACK, StackValue); - L1->tbclist = L1->stack; + L1->stack.p = luaM_newvector(L, BASIC_STACK_SIZE + EXTRA_STACK, StackValue); + L1->tbclist.p = L1->stack.p; for (i = 0; i < BASIC_STACK_SIZE + EXTRA_STACK; i++) - setnilvalue(s2v(L1->stack + i)); /* erase new stack */ - L1->top = L1->stack; - L1->stack_last = L1->stack + BASIC_STACK_SIZE; + setnilvalue(s2v(L1->stack.p + i)); /* erase new stack */ + L1->top.p = L1->stack.p; + L1->stack_last.p = L1->stack.p + BASIC_STACK_SIZE; /* initialize first ci */ ci = &L1->base_ci; ci->next = ci->previous = NULL; ci->callstatus = CIST_C; - ci->func = L1->top; + ci->func.p = L1->top.p; ci->u.c.k = NULL; ci->nresults = 0; - setnilvalue(s2v(L1->top)); /* 'function' entry for this 'ci' */ - L1->top++; - ci->top = L1->top + LUA_MINSTACK; + setnilvalue(s2v(L1->top.p)); /* 'function' entry for this 'ci' */ + L1->top.p++; + ci->top.p = L1->top.p + LUA_MINSTACK; L1->ci = ci; } static void freestack (lua_State *L) { - if (L->stack == NULL) + if (L->stack.p == NULL) return; /* stack not completely built yet */ L->ci = &L->base_ci; /* free the entire 'ci' list */ luaE_freeCI(L); lua_assert(L->nci == 0); - luaM_freearray(L, L->stack, stacksize(L) + EXTRA_STACK); /* free stack */ + luaM_freearray(L, L->stack.p, stacksize(L) + EXTRA_STACK); /* free stack */ } @@ -248,7 +248,7 @@ static void f_luaopen (lua_State *L, void *ud) { */ static void preinit_thread (lua_State *L, global_State *g) { G(L) = g; - L->stack = NULL; + L->stack.p = NULL; L->ci = NULL; L->nci = 0; L->twups = L; /* thread has no upvalues */ @@ -284,20 +284,16 @@ static void close_state (lua_State *L) { LUA_API lua_State *lua_newthread (lua_State *L) { - global_State *g; + global_State *g = G(L); + GCObject *o; lua_State *L1; lua_lock(L); - g = G(L); luaC_checkGC(L); /* create new thread */ - L1 = &cast(LX *, luaM_newobject(L, LUA_TTHREAD, sizeof(LX)))->l; - L1->marked = luaC_white(g); - L1->tt = LUA_VTHREAD; - /* link it on list 'allgc' */ - L1->next = g->allgc; - g->allgc = obj2gco(L1); + o = luaC_newobjdt(L, LUA_TTHREAD, sizeof(LX), offsetof(LX, l)); + L1 = gco2th(o); /* anchor it on L stack */ - setthvalue2s(L, L->top, L1); + setthvalue2s(L, L->top.p, L1); api_incr_top(L); preinit_thread(L1, g); L1->hookmask = L->hookmask; @@ -316,7 +312,7 @@ LUA_API lua_State *lua_newthread (lua_State *L) { void luaE_freethread (lua_State *L, lua_State *L1) { LX *l = fromstate(L1); - luaF_closeupval(L1, L1->stack); /* close all upvalues */ + luaF_closeupval(L1, L1->stack.p); /* close all upvalues */ lua_assert(L1->openupval == NULL); luai_userstatefree(L, L1); freestack(L1); @@ -326,26 +322,27 @@ void luaE_freethread (lua_State *L, lua_State *L1) { int luaE_resetthread (lua_State *L, int status) { CallInfo *ci = L->ci = &L->base_ci; /* unwind CallInfo list */ - setnilvalue(s2v(L->stack)); /* 'function' entry for basic 'ci' */ - ci->func = L->stack; + setnilvalue(s2v(L->stack.p)); /* 'function' entry for basic 'ci' */ + ci->func.p = L->stack.p; ci->callstatus = CIST_C; if (status == LUA_YIELD) status = LUA_OK; L->status = LUA_OK; /* so it can run __close metamethods */ status = luaD_closeprotected(L, 1, status); if (status != LUA_OK) /* errors? */ - luaD_seterrorobj(L, status, L->stack + 1); + luaD_seterrorobj(L, status, L->stack.p + 1); else - L->top = L->stack + 1; - ci->top = L->top + LUA_MINSTACK; - luaD_reallocstack(L, cast_int(ci->top - L->stack), 0); + L->top.p = L->stack.p + 1; + ci->top.p = L->top.p + LUA_MINSTACK; + luaD_reallocstack(L, cast_int(ci->top.p - L->stack.p), 0); return status; } -LUA_API int lua_resetthread (lua_State *L) { +LUA_API int lua_resetthread (lua_State *L, lua_State *from) { int status; lua_lock(L); + L->nCcalls = (from) ? getCcalls(from) : 0; status = luaE_resetthread(L, L->status); lua_unlock(L); return status; @@ -426,7 +423,7 @@ void luaE_warning (lua_State *L, const char *msg, int tocont) { ** Generate a warning from an error message */ void luaE_warnerror (lua_State *L, const char *where) { - TValue *errobj = s2v(L->top - 1); /* error object */ + TValue *errobj = s2v(L->top.p - 1); /* error object */ const char *msg = (ttisstring(errobj)) ? svalue(errobj) : "error object is not a string"; diff --git a/src/lua/lstate.h b/src/lua/lstate.h index 61e82cde7..8bf6600e3 100644 --- a/src/lua/lstate.h +++ b/src/lua/lstate.h @@ -9,6 +9,11 @@ #include "lua.h" + +/* Some header files included here need this definition */ +typedef struct CallInfo CallInfo; + + #include "lobject.h" #include "ltm.h" #include "lzio.h" @@ -139,7 +144,7 @@ struct lua_longjmp; /* defined in ldo.c */ #define BASIC_STACK_SIZE (2*LUA_MINSTACK) -#define stacksize(th) cast_int((th)->stack_last - (th)->stack) +#define stacksize(th) cast_int((th)->stack_last.p - (th)->stack.p) /* kinds of Garbage Collection */ @@ -169,9 +174,9 @@ typedef struct stringtable { ** - field 'transferinfo' is used only during call/returnhooks, ** before the function starts or after it ends. */ -typedef struct CallInfo { - StkId func; /* function index in the stack */ - StkId top; /* top for this function */ +struct CallInfo { + StkIdRel func; /* function index in the stack */ + StkIdRel top; /* top for this function */ struct CallInfo *previous, *next; /* dynamic call link */ union { struct { /* only for Lua functions */ @@ -196,7 +201,7 @@ typedef struct CallInfo { } u2; short nresults; /* expected number of results from this function */ unsigned short callstatus; -} CallInfo; +}; /* @@ -291,7 +296,7 @@ typedef struct global_State { struct lua_State *mainthread; TString *memerrmsg; /* message for memory-allocation errors */ TString *tmname[TM_N]; /* array with tag-method names */ - struct Table *mt[LUA_NUMTAGS]; /* metatables for basic types */ + struct Table *mt[LUA_NUMTYPES]; /* metatables for basic types */ TString *strcache[STRCACHE_N][STRCACHE_M]; /* cache for strings in API */ lua_WarnFunction warnf; /* warning function */ void *ud_warn; /* auxiliary data to 'warnf' */ @@ -306,13 +311,13 @@ struct lua_State { lu_byte status; lu_byte allowhook; unsigned short nci; /* number of items in 'ci' list */ - StkId top; /* first free slot in the stack */ + StkIdRel top; /* first free slot in the stack */ global_State *l_G; CallInfo *ci; /* call info for current function */ - StkId stack_last; /* end of stack (last element + 1) */ - StkId stack; /* stack base */ + StkIdRel stack_last; /* end of stack (last element + 1) */ + StkIdRel stack; /* stack base */ UpVal *openupval; /* list of open upvalues in this stack */ - StkId tbclist; /* list of to-be-closed variables */ + StkIdRel tbclist; /* list of to-be-closed variables */ GCObject *gclist; struct lua_State *twups; /* list of threads with open upvalues */ struct lua_longjmp *errorJmp; /* current error recover point */ diff --git a/src/lua/lstrlib.c b/src/lua/lstrlib.c index 0b4fdbb7b..03167161d 100644 --- a/src/lua/lstrlib.c +++ b/src/lua/lstrlib.c @@ -570,7 +570,7 @@ static const char *match_capture (MatchState *ms, const char *s, int l) { static const char *match (MatchState *ms, const char *s, const char *p) { if (l_unlikely(ms->matchdepth-- == 0)) luaL_error(ms->L, "pattern too complex"); - init: /* using goto's to optimize tail recursion */ + init: /* using goto to optimize tail recursion */ if (p != ms->p_end) { /* end of pattern? */ switch (*p) { case '(': { /* start capture */ diff --git a/src/lua/ltable.c b/src/lua/ltable.c index 1b1cd2415..3c690c5f1 100644 --- a/src/lua/ltable.c +++ b/src/lua/ltable.c @@ -107,7 +107,7 @@ static const TValue absentkey = {ABSTKEYCONSTANT}; */ static Node *hashint (const Table *t, lua_Integer i) { lua_Unsigned ui = l_castS2U(i); - if (ui <= (unsigned int)INT_MAX) + if (ui <= cast_uint(INT_MAX)) return hashmod(t, cast_int(ui)); else return hashmod(t, ui); @@ -257,9 +257,11 @@ LUAI_FUNC unsigned int luaH_realasize (const Table *t) { size |= (size >> 2); size |= (size >> 4); size |= (size >> 8); +#if (UINT_MAX >> 14) > 3 /* unsigned int has more than 16 bits */ size |= (size >> 16); #if (UINT_MAX >> 30) > 3 size |= (size >> 32); /* unsigned int has more than 32 bits */ +#endif #endif size++; lua_assert(ispow2(size) && size/2 < t->alimit && t->alimit < size); @@ -488,7 +490,7 @@ static void setnodevector (lua_State *L, Table *t, unsigned int size) { luaG_runerror(L, "table overflow"); size = twoto(lsize); t->node = luaM_newvector(L, size, Node); - for (i = 0; i < (int)size; i++) { + for (i = 0; i < cast_int(size); i++) { Node *n = gnode(t, i); gnext(n) = 0; setnilkey(n); @@ -975,6 +977,4 @@ Node *luaH_mainposition (const Table *t, const TValue *key) { return mainpositionTV(t, key); } -int luaH_isdummy (const Table *t) { return isdummy(t); } - #endif diff --git a/src/lua/ltable.h b/src/lua/ltable.h index 7bbbcb213..75dd9e26e 100644 --- a/src/lua/ltable.h +++ b/src/lua/ltable.h @@ -59,7 +59,6 @@ LUAI_FUNC unsigned int luaH_realasize (const Table *t); #if defined(LUA_DEBUG) LUAI_FUNC Node *luaH_mainposition (const Table *t, const TValue *key); -LUAI_FUNC int luaH_isdummy (const Table *t); #endif diff --git a/src/lua/ltablib.c b/src/lua/ltablib.c index 868d78fd8..e6bc4d04a 100644 --- a/src/lua/ltablib.c +++ b/src/lua/ltablib.c @@ -93,7 +93,7 @@ static int tremove (lua_State *L) { lua_Integer pos = luaL_optinteger(L, 2, size); if (pos != size) /* validate 'pos' if given */ /* check whether 'pos' is in [1, size + 1] */ - luaL_argcheck(L, (lua_Unsigned)pos - 1u <= (lua_Unsigned)size, 1, + luaL_argcheck(L, (lua_Unsigned)pos - 1u <= (lua_Unsigned)size, 2, "position out of bounds"); lua_geti(L, 1, pos); /* result = t[pos] */ for ( ; pos < size; pos++) { diff --git a/src/lua/ltm.c b/src/lua/ltm.c index b657b783a..07a060811 100644 --- a/src/lua/ltm.c +++ b/src/lua/ltm.c @@ -102,12 +102,12 @@ const char *luaT_objtypename (lua_State *L, const TValue *o) { void luaT_callTM (lua_State *L, const TValue *f, const TValue *p1, const TValue *p2, const TValue *p3) { - StkId func = L->top; + StkId func = L->top.p; setobj2s(L, func, f); /* push function (assume EXTRA_STACK) */ setobj2s(L, func + 1, p1); /* 1st argument */ setobj2s(L, func + 2, p2); /* 2nd argument */ setobj2s(L, func + 3, p3); /* 3rd argument */ - L->top = func + 4; + L->top.p = func + 4; /* metamethod may yield only when called from Lua code */ if (isLuacode(L->ci)) luaD_call(L, func, 0); @@ -119,18 +119,18 @@ void luaT_callTM (lua_State *L, const TValue *f, const TValue *p1, void luaT_callTMres (lua_State *L, const TValue *f, const TValue *p1, const TValue *p2, StkId res) { ptrdiff_t result = savestack(L, res); - StkId func = L->top; + StkId func = L->top.p; setobj2s(L, func, f); /* push function (assume EXTRA_STACK) */ setobj2s(L, func + 1, p1); /* 1st argument */ setobj2s(L, func + 2, p2); /* 2nd argument */ - L->top += 3; + L->top.p += 3; /* metamethod may yield only when called from Lua code */ if (isLuacode(L->ci)) luaD_call(L, func, 1); else luaD_callnoyield(L, func, 1); res = restorestack(L, result); - setobjs2s(L, res, --L->top); /* move result to its place */ + setobjs2s(L, res, --L->top.p); /* move result to its place */ } @@ -165,7 +165,7 @@ void luaT_trybinTM (lua_State *L, const TValue *p1, const TValue *p2, void luaT_tryconcatTM (lua_State *L) { - StkId top = L->top; + StkId top = L->top.p; if (l_unlikely(!callbinTM(L, s2v(top - 2), s2v(top - 1), top - 2, TM_CONCAT))) luaG_concaterror(L, s2v(top - 2), s2v(top - 1)); @@ -200,15 +200,15 @@ void luaT_trybiniTM (lua_State *L, const TValue *p1, lua_Integer i2, */ int luaT_callorderTM (lua_State *L, const TValue *p1, const TValue *p2, TMS event) { - if (callbinTM(L, p1, p2, L->top, event)) /* try original event */ - return !l_isfalse(s2v(L->top)); + if (callbinTM(L, p1, p2, L->top.p, event)) /* try original event */ + return !l_isfalse(s2v(L->top.p)); #if defined(LUA_COMPAT_LT_LE) else if (event == TM_LE) { /* try '!(p2 < p1)' for '(p1 <= p2)' */ L->ci->callstatus |= CIST_LEQ; /* mark it is doing 'lt' for 'le' */ - if (callbinTM(L, p2, p1, L->top, TM_LT)) { + if (callbinTM(L, p2, p1, L->top.p, TM_LT)) { L->ci->callstatus ^= CIST_LEQ; /* clear mark */ - return l_isfalse(s2v(L->top)); + return l_isfalse(s2v(L->top.p)); } /* else error will remove this 'ci'; no need to clear mark */ } @@ -238,20 +238,20 @@ int luaT_callorderiTM (lua_State *L, const TValue *p1, int v2, void luaT_adjustvarargs (lua_State *L, int nfixparams, CallInfo *ci, const Proto *p) { int i; - int actual = cast_int(L->top - ci->func) - 1; /* number of arguments */ + int actual = cast_int(L->top.p - ci->func.p) - 1; /* number of arguments */ int nextra = actual - nfixparams; /* number of extra arguments */ ci->u.l.nextraargs = nextra; luaD_checkstack(L, p->maxstacksize + 1); /* copy function to the top of the stack */ - setobjs2s(L, L->top++, ci->func); + setobjs2s(L, L->top.p++, ci->func.p); /* move fixed parameters to the top of the stack */ for (i = 1; i <= nfixparams; i++) { - setobjs2s(L, L->top++, ci->func + i); - setnilvalue(s2v(ci->func + i)); /* erase original parameter (for GC) */ + setobjs2s(L, L->top.p++, ci->func.p + i); + setnilvalue(s2v(ci->func.p + i)); /* erase original parameter (for GC) */ } - ci->func += actual + 1; - ci->top += actual + 1; - lua_assert(L->top <= ci->top && ci->top <= L->stack_last); + ci->func.p += actual + 1; + ci->top.p += actual + 1; + lua_assert(L->top.p <= ci->top.p && ci->top.p <= L->stack_last.p); } @@ -261,10 +261,10 @@ void luaT_getvarargs (lua_State *L, CallInfo *ci, StkId where, int wanted) { if (wanted < 0) { wanted = nextra; /* get all extra arguments available */ checkstackGCp(L, nextra, where); /* ensure stack space */ - L->top = where + nextra; /* next instruction will need top */ + L->top.p = where + nextra; /* next instruction will need top */ } for (i = 0; i < wanted && i < nextra; i++) - setobjs2s(L, where + i, ci->func - nextra + i); + setobjs2s(L, where + i, ci->func.p - nextra + i); for (; i < wanted; i++) /* complete required results with nil */ setnilvalue(s2v(where + i)); } diff --git a/src/lua/ltm.h b/src/lua/ltm.h index 73b833c60..c309e2ae1 100644 --- a/src/lua/ltm.h +++ b/src/lua/ltm.h @@ -9,6 +9,7 @@ #include "lobject.h" +#include "lstate.h" /* @@ -95,8 +96,8 @@ LUAI_FUNC int luaT_callorderiTM (lua_State *L, const TValue *p1, int v2, int inv, int isfloat, TMS event); LUAI_FUNC void luaT_adjustvarargs (lua_State *L, int nfixparams, - struct CallInfo *ci, const Proto *p); -LUAI_FUNC void luaT_getvarargs (lua_State *L, struct CallInfo *ci, + CallInfo *ci, const Proto *p); +LUAI_FUNC void luaT_getvarargs (lua_State *L, CallInfo *ci, StkId where, int wanted); diff --git a/src/lua/lua.c b/src/lua/lua.c index 03815a054..0ff884545 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -20,10 +20,6 @@ #include "lauxlib.h" #include "lualib.h" -/** Pi-hole modification **/ -#include "ftl_lua.h" -/**************************/ - #if !defined(LUA_PROGNAME) #define LUA_PROGNAME "lua" @@ -181,10 +177,11 @@ static void print_version (void) { ** to the script (everything after 'script') go to positive indices; ** other arguments (before the script name) go to negative indices. ** If there is no script name, assume interpreter's name as base. +** (If there is no interpreter's name either, 'script' is -1, so +** table sizes are zero.) */ static void createargtable (lua_State *L, char **argv, int argc, int script) { int i, narg; - if (script == argc) script = 0; /* no script name? */ narg = argc - (script + 1); /* number of positive indices */ lua_createtable(L, narg, script + 1); for (i = 0; i < argc; i++) { @@ -214,9 +211,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { /* ** Receives 'globname[=modname]' and runs 'globname = require(modname)'. */ -/************** Pi-hole modification ***************/ -int dolibrary (lua_State *L, char *globname) { -/***************************************************/ +static int dolibrary (lua_State *L, char *globname) { int status; char *modname = strchr(globname, '='); if (modname == NULL) /* no explicit name? */ @@ -274,14 +269,23 @@ static int handle_script (lua_State *L, char **argv) { /* ** Traverses all arguments from 'argv', returning a mask with those -** needed before running any Lua code (or an error code if it finds -** any invalid argument). 'first' returns the first not-handled argument -** (either the script name or a bad argument in case of error). +** needed before running any Lua code or an error code if it finds any +** invalid argument. In case of error, 'first' is the index of the bad +** argument. Otherwise, 'first' is -1 if there is no program name, +** 0 if there is no script name, or the index of the script name. */ static int collectargs (char **argv, int *first) { int args = 0; int i; - for (i = 1; argv[i] != NULL; i++) { + if (argv[0] != NULL) { /* is there a program name? */ + if (argv[0][0]) /* not empty? */ + progname = argv[0]; /* save it */ + } + else { /* no program name */ + *first = -1; + return 0; + } + for (i = 1; argv[i] != NULL; i++) { /* handle arguments */ *first = i; if (argv[i][0] != '-') /* not an option? */ return args; /* stop handling options */ @@ -322,7 +326,7 @@ static int collectargs (char **argv, int *first) { return has_error; } } - *first = i; /* no script name */ + *first = 0; /* no script name */ return args; } @@ -615,8 +619,8 @@ static int pmain (lua_State *L) { char **argv = (char **)lua_touserdata(L, 2); int script; int args = collectargs(argv, &script); + int optlim = (script > 0) ? script : argc; /* first argv not an option */ luaL_checkversion(L); /* check that interpreter has correct version */ - if (argv[0] && argv[0][0]) progname = argv[0]; if (args == has_error) { /* bad arg? */ print_usage(argv[script]); /* 'script' has index of bad arg. */ return 0; @@ -629,25 +633,21 @@ static int pmain (lua_State *L) { } luaL_openlibs(L); /* open standard libraries */ createargtable(L, argv, argc, script); /* create table 'arg' */ - lua_gc(L, LUA_GCGEN, 0, 0); /* GC in generational mode */ + lua_gc(L, LUA_GCRESTART); /* start GC... */ + lua_gc(L, LUA_GCGEN, 0, 0); /* ...in generational mode */ if (!(args & has_E)) { /* no option '-E'? */ if (handle_luainit(L) != LUA_OK) /* run LUA_INIT */ return 0; /* error running LUA_INIT */ } - - /************** Pi-hole modification ***************/ - // Load and enable libraries bundled with Pi-hole - ftl_lua_init(L); - /***************************************************/ - - if (!runargs(L, argv, script)) /* execute arguments -e and -l */ + if (!runargs(L, argv, optlim)) /* execute arguments -e and -l */ return 0; /* something failed */ - if (script < argc && /* execute main script (if there is one) */ - handle_script(L, argv + script) != LUA_OK) - return 0; + if (script > 0) { /* execute main script (if there is one) */ + if (handle_script(L, argv + script) != LUA_OK) + return 0; /* interrupt in case of error */ + } if (args & has_i) /* -i option? */ doREPL(L); /* do read-eval-print loop */ - else if (script == argc && !(args & (has_e | has_v))) { /* no arguments? */ + else if (script < 1 && !(args & (has_e | has_v))) { /* no active option? */ if (lua_stdin_is_tty()) { /* running in interactive mode? */ print_version(); doREPL(L); /* do read-eval-print loop */ @@ -659,15 +659,14 @@ static int pmain (lua_State *L) { } -/******* Pi-hole modification ********/ -int lua_main (int argc, char **argv) { -/*************************************/ +int main (int argc, char **argv) { int status, result; lua_State *L = luaL_newstate(); /* create state */ if (L == NULL) { l_message(argv[0], "cannot create state: not enough memory"); return EXIT_FAILURE; } + lua_gc(L, LUA_GCSTOP); /* stop GC while building state */ lua_pushcfunction(L, &pmain); /* to call 'pmain' in protected mode */ lua_pushinteger(L, argc); /* 1st argument */ lua_pushlightuserdata(L, argv); /* 2nd argument */ diff --git a/src/lua/lua.h b/src/lua/lua.h index e6618392c..01927c6d9 100644 --- a/src/lua/lua.h +++ b/src/lua/lua.h @@ -18,14 +18,14 @@ #define LUA_VERSION_MAJOR "5" #define LUA_VERSION_MINOR "4" -#define LUA_VERSION_RELEASE "4" +#define LUA_VERSION_RELEASE "5" #define LUA_VERSION_NUM 504 -#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 4) +#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 5) #define LUA_VERSION "Lua " LUA_VERSION_MAJOR "." LUA_VERSION_MINOR #define LUA_RELEASE LUA_VERSION "." LUA_VERSION_RELEASE -#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2022 Lua.org, PUC-Rio" +#define LUA_COPYRIGHT LUA_RELEASE " Copyright (C) 1994-2023 Lua.org, PUC-Rio" #define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo, W. Celes" @@ -131,6 +131,16 @@ typedef void * (*lua_Alloc) (void *ud, void *ptr, size_t osize, size_t nsize); typedef void (*lua_WarnFunction) (void *ud, const char *msg, int tocont); +/* +** Type used by the debug API to collect debug information +*/ +typedef struct lua_Debug lua_Debug; + + +/* +** Functions to be called by the debugger in specific events +*/ +typedef void (*lua_Hook) (lua_State *L, lua_Debug *ar); /* @@ -153,7 +163,7 @@ extern const char lua_ident[]; LUA_API lua_State *(lua_newstate) (lua_Alloc f, void *ud); LUA_API void (lua_close) (lua_State *L); LUA_API lua_State *(lua_newthread) (lua_State *L); -LUA_API int (lua_resetthread) (lua_State *L); +LUA_API int (lua_resetthread) (lua_State *L, lua_State *from); LUA_API lua_CFunction (lua_atpanic) (lua_State *L, lua_CFunction panicf); @@ -442,12 +452,6 @@ LUA_API void (lua_closeslot) (lua_State *L, int idx); #define LUA_MASKLINE (1 << LUA_HOOKLINE) #define LUA_MASKCOUNT (1 << LUA_HOOKCOUNT) -typedef struct lua_Debug lua_Debug; /* activation record */ - - -/* Functions to be called by the debugger in specific events */ -typedef void (*lua_Hook) (lua_State *L, lua_Debug *ar); - LUA_API int (lua_getstack) (lua_State *L, int level, lua_Debug *ar); LUA_API int (lua_getinfo) (lua_State *L, const char *what, lua_Debug *ar); @@ -492,7 +496,7 @@ struct lua_Debug { /****************************************************************************** -* Copyright (C) 1994-2022 Lua.org, PUC-Rio. +* Copyright (C) 1994-2023 Lua.org, PUC-Rio. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the diff --git a/src/lua/luac.c b/src/lua/luac.c index 6b556b267..5f4a141bd 100644 --- a/src/lua/luac.c +++ b/src/lua/luac.c @@ -121,7 +121,7 @@ static int doargs(int argc, char* argv[]) return i; } -#define FUNCTION "(function()end)();" +#define FUNCTION "(function()end)();\n" static const char* reader(lua_State* L, void* ud, size_t* size) { @@ -138,7 +138,7 @@ static const char* reader(lua_State* L, void* ud, size_t* size) } } -#define toproto(L,i) getproto(s2v(L->top+(i))) +#define toproto(L,i) getproto(s2v(L->top.p+(i))) static const Proto* combine(lua_State* L, int n) { @@ -155,8 +155,6 @@ static const Proto* combine(lua_State* L, int n) f->p[i]=toproto(L,i-n-1); if (f->p[i]->sizeupvalues>0) f->p[i]->upvalues[0].instack=0; } - luaM_freearray(L,f->lineinfo,f->sizelineinfo); - f->sizelineinfo=0; return f; } } @@ -195,9 +193,7 @@ static int pmain(lua_State* L) return 0; } -/******* Pi-hole modification ********/ -int luac_main(int argc, char* argv[]) -/*************************************/ +int main(int argc, char* argv[]) { lua_State* L; int i=doargs(argc,argv); diff --git a/src/lua/luaconf.h b/src/lua/luaconf.h index d42d14b7d..137103ede 100644 --- a/src/lua/luaconf.h +++ b/src/lua/luaconf.h @@ -70,6 +70,12 @@ #endif +#if defined(LUA_USE_IOS) +#define LUA_USE_POSIX +#define LUA_USE_DLOPEN +#endif + + /* @@ LUAI_IS32INT is true iff 'int' has (at least) 32 bits. */ @@ -728,7 +734,7 @@ ** CHANGE it if you need a different limit. This limit is arbitrary; ** its only purpose is to stop Lua from consuming unlimited stack ** space (and to reserve some numbers for pseudo-indices). -** (It must fit into max(size_t)/32.) +** (It must fit into max(size_t)/32 and max(int)/2.) */ #if LUAI_IS32INT #define LUAI_MAXSTACK 1000000 @@ -747,14 +753,15 @@ /* @@ LUA_IDSIZE gives the maximum size for the description of the source -@@ of a function in debug information. +** of a function in debug information. ** CHANGE it if you want a different size. */ #define LUA_IDSIZE 60 /* -@@ LUAL_BUFFERSIZE is the buffer size used by the lauxlib buffer system. +@@ LUAL_BUFFERSIZE is the initial buffer size used by the lauxlib +** buffer system. */ #define LUAL_BUFFERSIZE ((int)(16 * sizeof(void*) * sizeof(lua_Number))) diff --git a/src/lua/lualib.h b/src/lua/lualib.h index a7cab4eb0..262552907 100644 --- a/src/lua/lualib.h +++ b/src/lua/lualib.h @@ -44,10 +44,6 @@ LUAMOD_API int (luaopen_debug) (lua_State *L); #define LUA_LOADLIBNAME "package" LUAMOD_API int (luaopen_package) (lua_State *L); -/************ Pi-hole modification *************/ -#define LUA_PIHOLELIBNAME "pihole" -LUAMOD_API int (luaopen_pihole) (lua_State *L); -/***********************************************/ /* open all previous libraries */ LUALIB_API void (luaL_openlibs) (lua_State *L); diff --git a/src/lua/lundump.c b/src/lua/lundump.c index 5aa55c445..02aed64fb 100644 --- a/src/lua/lundump.c +++ b/src/lua/lundump.c @@ -120,10 +120,10 @@ static TString *loadStringN (LoadState *S, Proto *p) { } else { /* long string */ ts = luaS_createlngstrobj(L, size); /* create string */ - setsvalue2s(L, L->top, ts); /* anchor it ('loadVector' can GC) */ + setsvalue2s(L, L->top.p, ts); /* anchor it ('loadVector' can GC) */ luaD_inctop(L); loadVector(S, getstr(ts), size); /* load directly in final place */ - L->top--; /* pop string */ + L->top.p--; /* pop string */ } luaC_objbarrier(L, p, ts); return ts; @@ -248,6 +248,8 @@ static void loadDebug (LoadState *S, Proto *f) { f->locvars[i].endpc = loadInt(S); } n = loadInt(S); + if (n != 0) /* does it have debug information? */ + n = f->sizeupvalues; /* must be this many */ for (i = 0; i < n; i++) f->upvalues[i].name = loadStringN(S, f); } @@ -321,7 +323,7 @@ LClosure *luaU_undump(lua_State *L, ZIO *Z, const char *name) { S.Z = Z; checkHeader(&S); cl = luaF_newLclosure(L, loadByte(&S)); - setclLvalue2s(L, L->top, cl); + setclLvalue2s(L, L->top.p, cl); luaD_inctop(L); cl->p = luaF_newproto(L); luaC_objbarrier(L, cl, cl->p); diff --git a/src/lua/lutf8lib.c b/src/lua/lutf8lib.c index e7bf098f6..3a5b9bc38 100644 --- a/src/lua/lutf8lib.c +++ b/src/lua/lutf8lib.c @@ -25,6 +25,9 @@ #define MAXUTF 0x7FFFFFFFu + +#define MSGInvalid "invalid UTF-8 code" + /* ** Integer type for decoded UTF-8 values; MAXUTF needs 31 bits. */ @@ -35,7 +38,8 @@ typedef unsigned long utfint; #endif -#define iscont(p) ((*(p) & 0xC0) == 0x80) +#define iscont(c) (((c) & 0xC0) == 0x80) +#define iscontp(p) iscont(*(p)) /* from strlib */ @@ -65,7 +69,7 @@ static const char *utf8_decode (const char *s, utfint *val, int strict) { int count = 0; /* to count number of continuation bytes */ for (; c & 0x40; c <<= 1) { /* while it needs continuation bytes... */ unsigned int cc = (unsigned char)s[++count]; /* read next byte */ - if ((cc & 0xC0) != 0x80) /* not a continuation byte? */ + if (!iscont(cc)) /* not a continuation byte? */ return NULL; /* invalid byte sequence */ res = (res << 6) | (cc & 0x3F); /* add lower 6 bits from cont. byte */ } @@ -140,7 +144,7 @@ static int codepoint (lua_State *L) { utfint code; s = utf8_decode(s, &code, !lax); if (s == NULL) - return luaL_error(L, "invalid UTF-8 code"); + return luaL_error(L, MSGInvalid); lua_pushinteger(L, code); n++; } @@ -190,16 +194,16 @@ static int byteoffset (lua_State *L) { "position out of bounds"); if (n == 0) { /* find beginning of current byte sequence */ - while (posi > 0 && iscont(s + posi)) posi--; + while (posi > 0 && iscontp(s + posi)) posi--; } else { - if (iscont(s + posi)) + if (iscontp(s + posi)) return luaL_error(L, "initial position is a continuation byte"); if (n < 0) { while (n < 0 && posi > 0) { /* move back */ do { /* find beginning of previous character */ posi--; - } while (posi > 0 && iscont(s + posi)); + } while (posi > 0 && iscontp(s + posi)); n++; } } @@ -208,7 +212,7 @@ static int byteoffset (lua_State *L) { while (n > 0 && posi < (lua_Integer)len) { do { /* find beginning of next character */ posi++; - } while (iscont(s + posi)); /* (cannot pass final '\0') */ + } while (iscontp(s + posi)); /* (cannot pass final '\0') */ n--; } } @@ -226,15 +230,15 @@ static int iter_aux (lua_State *L, int strict) { const char *s = luaL_checklstring(L, 1, &len); lua_Unsigned n = (lua_Unsigned)lua_tointeger(L, 2); if (n < len) { - while (iscont(s + n)) n++; /* skip continuation bytes */ + while (iscontp(s + n)) n++; /* go to next character */ } if (n >= len) /* (also handles original 'n' being negative) */ return 0; /* no more codepoints */ else { utfint code; const char *next = utf8_decode(s + n, &code, strict); - if (next == NULL) - return luaL_error(L, "invalid UTF-8 code"); + if (next == NULL || iscontp(next)) + return luaL_error(L, MSGInvalid); lua_pushinteger(L, n + 1); lua_pushinteger(L, code); return 2; @@ -253,7 +257,8 @@ static int iter_auxlax (lua_State *L) { static int iter_codes (lua_State *L) { int lax = lua_toboolean(L, 2); - luaL_checkstring(L, 1); + const char *s = luaL_checkstring(L, 1); + luaL_argcheck(L, !iscontp(s), 1, MSGInvalid); lua_pushcfunction(L, lax ? iter_auxlax : iter_auxstrict); lua_pushvalue(L, 1); lua_pushinteger(L, 0); diff --git a/src/lua/lvm.c b/src/lua/lvm.c index 2ec344003..8493a770c 100644 --- a/src/lua/lvm.c +++ b/src/lua/lvm.c @@ -608,8 +608,8 @@ int luaV_equalobj (lua_State *L, const TValue *t1, const TValue *t2) { if (tm == NULL) /* no TM? */ return 0; /* objects are different */ else { - luaT_callTMres(L, tm, t1, t2, L->top); /* call TM */ - return !l_isfalse(s2v(L->top)); + luaT_callTMres(L, tm, t1, t2, L->top.p); /* call TM */ + return !l_isfalse(s2v(L->top.p)); } } @@ -633,17 +633,17 @@ static void copy2buff (StkId top, int n, char *buff) { /* ** Main operation for concatenation: concat 'total' values in the stack, -** from 'L->top - total' up to 'L->top - 1'. +** from 'L->top.p - total' up to 'L->top.p - 1'. */ void luaV_concat (lua_State *L, int total) { if (total == 1) return; /* "all" values already concatenated */ do { - StkId top = L->top; + StkId top = L->top.p; int n = 2; /* number of elements handled in this pass (at least 2) */ if (!(ttisstring(s2v(top - 2)) || cvt2str(s2v(top - 2))) || !tostring(L, s2v(top - 1))) - luaT_tryconcatTM(L); + luaT_tryconcatTM(L); /* may invalidate 'top' */ else if (isemptystr(s2v(top - 1))) /* second operand is empty? */ cast_void(tostring(L, s2v(top - 2))); /* result is first operand */ else if (isemptystr(s2v(top - 2))) { /* first operand is empty string? */ @@ -656,8 +656,10 @@ void luaV_concat (lua_State *L, int total) { /* collect total length and number of strings */ for (n = 1; n < total && tostring(L, s2v(top - n - 1)); n++) { size_t l = vslen(s2v(top - n - 1)); - if (l_unlikely(l >= (MAX_SIZE/sizeof(char)) - tl)) + if (l_unlikely(l >= (MAX_SIZE/sizeof(char)) - tl)) { + L->top.p = top - total; /* pop strings to avoid wasting stack */ luaG_runerror(L, "string length overflow"); + } tl += l; } if (tl <= LUAI_MAXSHORTLEN) { /* is result a short string? */ @@ -671,8 +673,8 @@ void luaV_concat (lua_State *L, int total) { } setsvalue2s(L, top - n, ts); /* create result */ } - total -= n-1; /* got 'n' strings to create 1 new */ - L->top -= n-1; /* popped 'n' strings and pushed one */ + total -= n - 1; /* got 'n' strings to create one new */ + L->top.p -= n - 1; /* popped 'n' strings and pushed one */ } while (total > 1); /* repeat until only 1 result left */ } @@ -763,12 +765,10 @@ lua_Number luaV_modf (lua_State *L, lua_Number m, lua_Number n) { /* number of bits in an integer */ #define NBITS cast_int(sizeof(lua_Integer) * CHAR_BIT) + /* ** Shift left operation. (Shift right just negates 'y'.) */ -#define luaV_shiftr(x,y) luaV_shiftl(x,intop(-, 0, y)) - - lua_Integer luaV_shiftl (lua_Integer x, lua_Integer y) { if (y < 0) { /* shift right? */ if (y <= -NBITS) return 0; @@ -808,26 +808,26 @@ static void pushclosure (lua_State *L, Proto *p, UpVal **encup, StkId base, */ void luaV_finishOp (lua_State *L) { CallInfo *ci = L->ci; - StkId base = ci->func + 1; + StkId base = ci->func.p + 1; Instruction inst = *(ci->u.l.savedpc - 1); /* interrupted instruction */ OpCode op = GET_OPCODE(inst); switch (op) { /* finish its execution */ case OP_MMBIN: case OP_MMBINI: case OP_MMBINK: { - setobjs2s(L, base + GETARG_A(*(ci->u.l.savedpc - 2)), --L->top); + setobjs2s(L, base + GETARG_A(*(ci->u.l.savedpc - 2)), --L->top.p); break; } case OP_UNM: case OP_BNOT: case OP_LEN: case OP_GETTABUP: case OP_GETTABLE: case OP_GETI: case OP_GETFIELD: case OP_SELF: { - setobjs2s(L, base + GETARG_A(inst), --L->top); + setobjs2s(L, base + GETARG_A(inst), --L->top.p); break; } case OP_LT: case OP_LE: case OP_LTI: case OP_LEI: case OP_GTI: case OP_GEI: case OP_EQ: { /* note that 'OP_EQI'/'OP_EQK' cannot yield */ - int res = !l_isfalse(s2v(L->top - 1)); - L->top--; + int res = !l_isfalse(s2v(L->top.p - 1)); + L->top.p--; #if defined(LUA_COMPAT_LT_LE) if (ci->callstatus & CIST_LEQ) { /* "<=" using "<" instead? */ ci->callstatus ^= CIST_LEQ; /* clear mark */ @@ -840,11 +840,11 @@ void luaV_finishOp (lua_State *L) { break; } case OP_CONCAT: { - StkId top = L->top - 1; /* top when 'luaT_tryconcatTM' was called */ + StkId top = L->top.p - 1; /* top when 'luaT_tryconcatTM' was called */ int a = GETARG_A(inst); /* first element to concatenate */ int total = cast_int(top - 1 - (base + a)); /* yet to concatenate */ setobjs2s(L, top - 2, top); /* put TM result in proper position */ - L->top = top - 1; /* top is one after last element (at top-2) */ + L->top.p = top - 1; /* top is one after last element (at top-2) */ luaV_concat(L, total); /* concat them (may yield again) */ break; } @@ -856,7 +856,7 @@ void luaV_finishOp (lua_State *L) { StkId ra = base + GETARG_A(inst); /* adjust top to signal correct number of returns, in case the return is "up to top" ('isIT') */ - L->top = ra + ci->u2.nres; + L->top.p = ra + ci->u2.nres; /* repeat instruction to close other vars. and complete the return */ ci->u.l.savedpc--; break; @@ -898,6 +898,7 @@ void luaV_finishOp (lua_State *L) { ** operation, 'fop' is the float operation. */ #define op_arithI(L,iop,fop) { \ + StkId ra = RA(i); \ TValue *v1 = vRB(i); \ int imm = GETARG_sC(i); \ if (ttisinteger(v1)) { \ @@ -926,6 +927,7 @@ void luaV_finishOp (lua_State *L) { ** Arithmetic operations over floats and others with register operands. */ #define op_arithf(L,fop) { \ + StkId ra = RA(i); \ TValue *v1 = vRB(i); \ TValue *v2 = vRC(i); \ op_arithf_aux(L, v1, v2, fop); } @@ -935,6 +937,7 @@ void luaV_finishOp (lua_State *L) { ** Arithmetic operations with K operands for floats. */ #define op_arithfK(L,fop) { \ + StkId ra = RA(i); \ TValue *v1 = vRB(i); \ TValue *v2 = KC(i); lua_assert(ttisnumber(v2)); \ op_arithf_aux(L, v1, v2, fop); } @@ -944,6 +947,7 @@ void luaV_finishOp (lua_State *L) { ** Arithmetic operations over integers and floats. */ #define op_arith_aux(L,v1,v2,iop,fop) { \ + StkId ra = RA(i); \ if (ttisinteger(v1) && ttisinteger(v2)) { \ lua_Integer i1 = ivalue(v1); lua_Integer i2 = ivalue(v2); \ pc++; setivalue(s2v(ra), iop(L, i1, i2)); \ @@ -973,6 +977,7 @@ void luaV_finishOp (lua_State *L) { ** Bitwise operations with constant operand. */ #define op_bitwiseK(L,op) { \ + StkId ra = RA(i); \ TValue *v1 = vRB(i); \ TValue *v2 = KC(i); \ lua_Integer i1; \ @@ -986,6 +991,7 @@ void luaV_finishOp (lua_State *L) { ** Bitwise operations with register operands. */ #define op_bitwise(L,op) { \ + StkId ra = RA(i); \ TValue *v1 = vRB(i); \ TValue *v2 = vRC(i); \ lua_Integer i1; lua_Integer i2; \ @@ -1000,18 +1006,19 @@ void luaV_finishOp (lua_State *L) { ** integers. */ #define op_order(L,opi,opn,other) { \ - int cond; \ - TValue *rb = vRB(i); \ - if (ttisinteger(s2v(ra)) && ttisinteger(rb)) { \ - lua_Integer ia = ivalue(s2v(ra)); \ - lua_Integer ib = ivalue(rb); \ - cond = opi(ia, ib); \ - } \ - else if (ttisnumber(s2v(ra)) && ttisnumber(rb)) \ - cond = opn(s2v(ra), rb); \ - else \ - Protect(cond = other(L, s2v(ra), rb)); \ - docondjump(); } + StkId ra = RA(i); \ + int cond; \ + TValue *rb = vRB(i); \ + if (ttisinteger(s2v(ra)) && ttisinteger(rb)) { \ + lua_Integer ia = ivalue(s2v(ra)); \ + lua_Integer ib = ivalue(rb); \ + cond = opi(ia, ib); \ + } \ + else if (ttisnumber(s2v(ra)) && ttisnumber(rb)) \ + cond = opn(s2v(ra), rb); \ + else \ + Protect(cond = other(L, s2v(ra), rb)); \ + docondjump(); } /* @@ -1019,20 +1026,21 @@ void luaV_finishOp (lua_State *L) { ** always small enough to have an exact representation as a float.) */ #define op_orderI(L,opi,opf,inv,tm) { \ - int cond; \ - int im = GETARG_sB(i); \ - if (ttisinteger(s2v(ra))) \ - cond = opi(ivalue(s2v(ra)), im); \ - else if (ttisfloat(s2v(ra))) { \ - lua_Number fa = fltvalue(s2v(ra)); \ - lua_Number fim = cast_num(im); \ - cond = opf(fa, fim); \ - } \ - else { \ - int isf = GETARG_C(i); \ - Protect(cond = luaT_callorderiTM(L, s2v(ra), im, inv, isf, tm)); \ - } \ - docondjump(); } + StkId ra = RA(i); \ + int cond; \ + int im = GETARG_sB(i); \ + if (ttisinteger(s2v(ra))) \ + cond = opi(ivalue(s2v(ra)), im); \ + else if (ttisfloat(s2v(ra))) { \ + lua_Number fa = fltvalue(s2v(ra)); \ + lua_Number fim = cast_num(im); \ + cond = opf(fa, fim); \ + } \ + else { \ + int isf = GETARG_C(i); \ + Protect(cond = luaT_callorderiTM(L, s2v(ra), im, inv, isf, tm)); \ + } \ + docondjump(); } /* }================================================================== */ @@ -1061,7 +1069,7 @@ void luaV_finishOp (lua_State *L) { #define updatetrap(ci) (trap = ci->u.l.trap) -#define updatebase(ci) (base = ci->func + 1) +#define updatebase(ci) (base = ci->func.p + 1) #define updatestack(ci) \ @@ -1096,7 +1104,7 @@ void luaV_finishOp (lua_State *L) { ** Whenever code can raise errors, the global 'pc' and the global ** 'top' must be correct to report occasional errors. */ -#define savestate(L,ci) (savepc(L), L->top = ci->top) +#define savestate(L,ci) (savepc(L), L->top.p = ci->top.p) /* @@ -1116,7 +1124,7 @@ void luaV_finishOp (lua_State *L) { /* 'c' is the limit of live values in the stack */ #define checkGC(L,c) \ - { luaC_condGC(L, (savepc(L), L->top = (c)), \ + { luaC_condGC(L, (savepc(L), L->top.p = (c)), \ updatetrap(ci)); \ luai_threadyield(L); } @@ -1128,7 +1136,6 @@ void luaV_finishOp (lua_State *L) { updatebase(ci); /* correct stack */ \ } \ i = *(pc++); \ - ra = RA(i); /* WARNING: any stack reallocation invalidates 'ra' */ \ } #define vmdispatch(o) switch(o) @@ -1148,7 +1155,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { startfunc: trap = L->hookmask; returning: /* trap already set */ - cl = clLvalue(s2v(ci->func)); + cl = clLvalue(s2v(ci->func.p)); k = cl->p->k; pc = ci->u.l.savedpc; if (l_unlikely(trap)) { @@ -1160,60 +1167,68 @@ void luaV_execute (lua_State *L, CallInfo *ci) { } ci->u.l.trap = 1; /* assume trap is on, for now */ } - base = ci->func + 1; + base = ci->func.p + 1; /* main loop of interpreter */ for (;;) { Instruction i; /* instruction being executed */ - StkId ra; /* instruction's A register */ vmfetch(); #if 0 /* low-level line tracing for debugging Lua */ printf("line: %d\n", luaG_getfuncline(cl->p, pcRel(pc, cl->p))); #endif - lua_assert(base == ci->func + 1); - lua_assert(base <= L->top && L->top < L->stack_last); + lua_assert(base == ci->func.p + 1); + lua_assert(base <= L->top.p && L->top.p <= L->stack_last.p); /* invalidate top for instructions not expecting it */ - lua_assert(isIT(i) || (cast_void(L->top = base), 1)); + lua_assert(isIT(i) || (cast_void(L->top.p = base), 1)); vmdispatch (GET_OPCODE(i)) { vmcase(OP_MOVE) { + StkId ra = RA(i); setobjs2s(L, ra, RB(i)); vmbreak; } vmcase(OP_LOADI) { + StkId ra = RA(i); lua_Integer b = GETARG_sBx(i); setivalue(s2v(ra), b); vmbreak; } vmcase(OP_LOADF) { + StkId ra = RA(i); int b = GETARG_sBx(i); setfltvalue(s2v(ra), cast_num(b)); vmbreak; } vmcase(OP_LOADK) { + StkId ra = RA(i); TValue *rb = k + GETARG_Bx(i); setobj2s(L, ra, rb); vmbreak; } vmcase(OP_LOADKX) { + StkId ra = RA(i); TValue *rb; rb = k + GETARG_Ax(*pc); pc++; setobj2s(L, ra, rb); vmbreak; } vmcase(OP_LOADFALSE) { + StkId ra = RA(i); setbfvalue(s2v(ra)); vmbreak; } vmcase(OP_LFALSESKIP) { + StkId ra = RA(i); setbfvalue(s2v(ra)); pc++; /* skip next instruction */ vmbreak; } vmcase(OP_LOADTRUE) { + StkId ra = RA(i); setbtvalue(s2v(ra)); vmbreak; } vmcase(OP_LOADNIL) { + StkId ra = RA(i); int b = GETARG_B(i); do { setnilvalue(s2v(ra++)); @@ -1221,19 +1236,22 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_GETUPVAL) { + StkId ra = RA(i); int b = GETARG_B(i); - setobj2s(L, ra, cl->upvals[b]->v); + setobj2s(L, ra, cl->upvals[b]->v.p); vmbreak; } vmcase(OP_SETUPVAL) { + StkId ra = RA(i); UpVal *uv = cl->upvals[GETARG_B(i)]; - setobj(L, uv->v, s2v(ra)); + setobj(L, uv->v.p, s2v(ra)); luaC_barrier(L, uv, s2v(ra)); vmbreak; } vmcase(OP_GETTABUP) { + StkId ra = RA(i); const TValue *slot; - TValue *upval = cl->upvals[GETARG_B(i)]->v; + TValue *upval = cl->upvals[GETARG_B(i)]->v.p; TValue *rc = KC(i); TString *key = tsvalue(rc); /* key must be a string */ if (luaV_fastget(L, upval, key, slot, luaH_getshortstr)) { @@ -1244,6 +1262,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_GETTABLE) { + StkId ra = RA(i); const TValue *slot; TValue *rb = vRB(i); TValue *rc = vRC(i); @@ -1258,6 +1277,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_GETI) { + StkId ra = RA(i); const TValue *slot; TValue *rb = vRB(i); int c = GETARG_C(i); @@ -1272,6 +1292,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_GETFIELD) { + StkId ra = RA(i); const TValue *slot; TValue *rb = vRB(i); TValue *rc = KC(i); @@ -1285,7 +1306,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { } vmcase(OP_SETTABUP) { const TValue *slot; - TValue *upval = cl->upvals[GETARG_A(i)]->v; + TValue *upval = cl->upvals[GETARG_A(i)]->v.p; TValue *rb = KB(i); TValue *rc = RKC(i); TString *key = tsvalue(rb); /* key must be a string */ @@ -1297,6 +1318,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_SETTABLE) { + StkId ra = RA(i); const TValue *slot; TValue *rb = vRB(i); /* key (table is in 'ra') */ TValue *rc = RKC(i); /* value */ @@ -1311,6 +1333,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_SETI) { + StkId ra = RA(i); const TValue *slot; int c = GETARG_B(i); TValue *rc = RKC(i); @@ -1325,6 +1348,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_SETFIELD) { + StkId ra = RA(i); const TValue *slot; TValue *rb = KB(i); TValue *rc = RKC(i); @@ -1337,6 +1361,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_NEWTABLE) { + StkId ra = RA(i); int b = GETARG_B(i); /* log2(hash size) + 1 */ int c = GETARG_C(i); /* array size */ Table *t; @@ -1346,7 +1371,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { if (TESTARG_k(i)) /* non-zero extra argument? */ c += GETARG_Ax(*pc) * (MAXARG_C + 1); /* add it to size */ pc++; /* skip extra argument */ - L->top = ra + 1; /* correct top in case of emergency GC */ + L->top.p = ra + 1; /* correct top in case of emergency GC */ t = luaH_new(L); /* memory allocation */ sethvalue2s(L, ra, t); if (b != 0 || c != 0) @@ -1355,6 +1380,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_SELF) { + StkId ra = RA(i); const TValue *slot; TValue *rb = vRB(i); TValue *rc = RKC(i); @@ -1384,6 +1410,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_MODK) { + savestate(L, ci); /* in case of division by 0 */ op_arithK(L, luaV_mod, luaV_modf); vmbreak; } @@ -1396,6 +1423,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_IDIVK) { + savestate(L, ci); /* in case of division by 0 */ op_arithK(L, luaV_idiv, luai_numidiv); vmbreak; } @@ -1412,6 +1440,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_SHRI) { + StkId ra = RA(i); TValue *rb = vRB(i); int ic = GETARG_sC(i); lua_Integer ib; @@ -1421,6 +1450,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_SHLI) { + StkId ra = RA(i); TValue *rb = vRB(i); int ic = GETARG_sC(i); lua_Integer ib; @@ -1442,6 +1472,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_MOD) { + savestate(L, ci); /* in case of division by 0 */ op_arith(L, luaV_mod, luaV_modf); vmbreak; } @@ -1454,6 +1485,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_IDIV) { /* floor division */ + savestate(L, ci); /* in case of division by 0 */ op_arith(L, luaV_idiv, luai_numidiv); vmbreak; } @@ -1478,6 +1510,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_MMBIN) { + StkId ra = RA(i); Instruction pi = *(pc - 2); /* original arith. expression */ TValue *rb = vRB(i); TMS tm = (TMS)GETARG_C(i); @@ -1487,6 +1520,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_MMBINI) { + StkId ra = RA(i); Instruction pi = *(pc - 2); /* original arith. expression */ int imm = GETARG_sB(i); TMS tm = (TMS)GETARG_C(i); @@ -1496,6 +1530,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_MMBINK) { + StkId ra = RA(i); Instruction pi = *(pc - 2); /* original arith. expression */ TValue *imm = KB(i); TMS tm = (TMS)GETARG_C(i); @@ -1505,6 +1540,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_UNM) { + StkId ra = RA(i); TValue *rb = vRB(i); lua_Number nb; if (ttisinteger(rb)) { @@ -1519,6 +1555,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_BNOT) { + StkId ra = RA(i); TValue *rb = vRB(i); lua_Integer ib; if (tointegerns(rb, &ib)) { @@ -1529,6 +1566,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_NOT) { + StkId ra = RA(i); TValue *rb = vRB(i); if (l_isfalse(rb)) setbtvalue(s2v(ra)); @@ -1537,21 +1575,25 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_LEN) { + StkId ra = RA(i); Protect(luaV_objlen(L, ra, vRB(i))); vmbreak; } vmcase(OP_CONCAT) { + StkId ra = RA(i); int n = GETARG_B(i); /* number of elements to concatenate */ - L->top = ra + n; /* mark the end of concat operands */ + L->top.p = ra + n; /* mark the end of concat operands */ ProtectNT(luaV_concat(L, n)); - checkGC(L, L->top); /* 'luaV_concat' ensures correct top */ + checkGC(L, L->top.p); /* 'luaV_concat' ensures correct top */ vmbreak; } vmcase(OP_CLOSE) { + StkId ra = RA(i); Protect(luaF_close(L, ra, LUA_OK, 1)); vmbreak; } vmcase(OP_TBC) { + StkId ra = RA(i); /* create new to-be-closed upvalue */ halfProtect(luaF_newtbcupval(L, ra)); vmbreak; @@ -1561,6 +1603,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_EQ) { + StkId ra = RA(i); int cond; TValue *rb = vRB(i); Protect(cond = luaV_equalobj(L, s2v(ra), rb)); @@ -1576,6 +1619,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_EQK) { + StkId ra = RA(i); TValue *rb = KB(i); /* basic types do not use '__eq'; we can use raw equality */ int cond = luaV_rawequalobj(s2v(ra), rb); @@ -1583,6 +1627,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_EQI) { + StkId ra = RA(i); int cond; int im = GETARG_sB(i); if (ttisinteger(s2v(ra))) @@ -1611,11 +1656,13 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_TEST) { + StkId ra = RA(i); int cond = !l_isfalse(s2v(ra)); docondjump(); vmbreak; } vmcase(OP_TESTSET) { + StkId ra = RA(i); TValue *rb = vRB(i); if (l_isfalse(rb) == GETARG_k(i)) pc++; @@ -1626,11 +1673,12 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_CALL) { + StkId ra = RA(i); CallInfo *newci; int b = GETARG_B(i); int nresults = GETARG_C(i) - 1; if (b != 0) /* fixed number of arguments? */ - L->top = ra + b; /* top signals number of arguments */ + L->top.p = ra + b; /* top signals number of arguments */ /* else previous instruction set top */ savepc(L); /* in case of errors */ if ((newci = luaD_precall(L, ra, nresults)) == NULL) @@ -1642,54 +1690,57 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_TAILCALL) { + StkId ra = RA(i); int b = GETARG_B(i); /* number of arguments + 1 (function) */ int n; /* number of results when calling a C function */ int nparams1 = GETARG_C(i); /* delta is virtual 'func' - real 'func' (vararg functions) */ int delta = (nparams1) ? ci->u.l.nextraargs + nparams1 : 0; if (b != 0) - L->top = ra + b; + L->top.p = ra + b; else /* previous instruction set top */ - b = cast_int(L->top - ra); + b = cast_int(L->top.p - ra); savepc(ci); /* several calls here can raise errors */ if (TESTARG_k(i)) { luaF_closeupval(L, base); /* close upvalues from current call */ - lua_assert(L->tbclist < base); /* no pending tbc variables */ - lua_assert(base == ci->func + 1); + lua_assert(L->tbclist.p < base); /* no pending tbc variables */ + lua_assert(base == ci->func.p + 1); } if ((n = luaD_pretailcall(L, ci, ra, b, delta)) < 0) /* Lua function? */ goto startfunc; /* execute the callee */ else { /* C function? */ - ci->func -= delta; /* restore 'func' (if vararg) */ + ci->func.p -= delta; /* restore 'func' (if vararg) */ luaD_poscall(L, ci, n); /* finish caller */ updatetrap(ci); /* 'luaD_poscall' can change hooks */ goto ret; /* caller returns after the tail call */ } } vmcase(OP_RETURN) { + StkId ra = RA(i); int n = GETARG_B(i) - 1; /* number of results */ int nparams1 = GETARG_C(i); if (n < 0) /* not fixed? */ - n = cast_int(L->top - ra); /* get what is available */ + n = cast_int(L->top.p - ra); /* get what is available */ savepc(ci); if (TESTARG_k(i)) { /* may there be open upvalues? */ ci->u2.nres = n; /* save number of returns */ - if (L->top < ci->top) - L->top = ci->top; + if (L->top.p < ci->top.p) + L->top.p = ci->top.p; luaF_close(L, base, CLOSEKTOP, 1); updatetrap(ci); updatestack(ci); } if (nparams1) /* vararg function? */ - ci->func -= ci->u.l.nextraargs + nparams1; - L->top = ra + n; /* set call for 'luaD_poscall' */ + ci->func.p -= ci->u.l.nextraargs + nparams1; + L->top.p = ra + n; /* set call for 'luaD_poscall' */ luaD_poscall(L, ci, n); updatetrap(ci); /* 'luaD_poscall' can change hooks */ goto ret; } vmcase(OP_RETURN0) { if (l_unlikely(L->hookmask)) { - L->top = ra; + StkId ra = RA(i); + L->top.p = ra; savepc(ci); luaD_poscall(L, ci, 0); /* no hurry... */ trap = 1; @@ -1697,15 +1748,16 @@ void luaV_execute (lua_State *L, CallInfo *ci) { else { /* do the 'poscall' here */ int nres; L->ci = ci->previous; /* back to caller */ - L->top = base - 1; + L->top.p = base - 1; for (nres = ci->nresults; l_unlikely(nres > 0); nres--) - setnilvalue(s2v(L->top++)); /* all results are nil */ + setnilvalue(s2v(L->top.p++)); /* all results are nil */ } goto ret; } vmcase(OP_RETURN1) { if (l_unlikely(L->hookmask)) { - L->top = ra + 1; + StkId ra = RA(i); + L->top.p = ra + 1; savepc(ci); luaD_poscall(L, ci, 1); /* no hurry... */ trap = 1; @@ -1714,12 +1766,13 @@ void luaV_execute (lua_State *L, CallInfo *ci) { int nres = ci->nresults; L->ci = ci->previous; /* back to caller */ if (nres == 0) - L->top = base - 1; /* asked for no results */ + L->top.p = base - 1; /* asked for no results */ else { + StkId ra = RA(i); setobjs2s(L, base - 1, ra); /* at least this result */ - L->top = base; + L->top.p = base; for (; l_unlikely(nres > 1); nres--) - setnilvalue(s2v(L->top++)); /* complete missing results */ + setnilvalue(s2v(L->top.p++)); /* complete missing results */ } } ret: /* return from a Lua function */ @@ -1731,6 +1784,7 @@ void luaV_execute (lua_State *L, CallInfo *ci) { } } vmcase(OP_FORLOOP) { + StkId ra = RA(i); if (ttisinteger(s2v(ra + 2))) { /* integer loop? */ lua_Unsigned count = l_castS2U(ivalue(s2v(ra + 1))); if (count > 0) { /* still more iterations? */ @@ -1749,12 +1803,14 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_FORPREP) { + StkId ra = RA(i); savestate(L, ci); /* in case of errors */ if (forprep(L, ra)) pc += GETARG_Bx(i) + 1; /* skip the loop */ vmbreak; } vmcase(OP_TFORPREP) { + StkId ra = RA(i); /* create to-be-closed upvalue (if needed) */ halfProtect(luaF_newtbcupval(L, ra + 3)); pc += GETARG_Bx(i); @@ -1763,7 +1819,8 @@ void luaV_execute (lua_State *L, CallInfo *ci) { goto l_tforcall; } vmcase(OP_TFORCALL) { - l_tforcall: + l_tforcall: { + StkId ra = RA(i); /* 'ra' has the iterator function, 'ra + 1' has the state, 'ra + 2' has the control variable, and 'ra + 3' has the to-be-closed variable. The call will use the stack after @@ -1771,29 +1828,31 @@ void luaV_execute (lua_State *L, CallInfo *ci) { */ /* push function, state, and control variable */ memcpy(ra + 4, ra, 3 * sizeof(*ra)); - L->top = ra + 4 + 3; + L->top.p = ra + 4 + 3; ProtectNT(luaD_call(L, ra + 4, GETARG_C(i))); /* do the call */ updatestack(ci); /* stack may have changed */ i = *(pc++); /* go to next instruction */ lua_assert(GET_OPCODE(i) == OP_TFORLOOP && ra == RA(i)); goto l_tforloop; - } + }} vmcase(OP_TFORLOOP) { - l_tforloop: + l_tforloop: { + StkId ra = RA(i); if (!ttisnil(s2v(ra + 4))) { /* continue loop? */ setobjs2s(L, ra + 2, ra + 4); /* save control variable */ pc -= GETARG_Bx(i); /* jump back */ } vmbreak; - } + }} vmcase(OP_SETLIST) { + StkId ra = RA(i); int n = GETARG_B(i); unsigned int last = GETARG_C(i); Table *h = hvalue(s2v(ra)); if (n == 0) - n = cast_int(L->top - ra) - 1; /* get up to the top */ + n = cast_int(L->top.p - ra) - 1; /* get up to the top */ else - L->top = ci->top; /* correct top in case of emergency GC */ + L->top.p = ci->top.p; /* correct top in case of emergency GC */ last += n; if (TESTARG_k(i)) { last += GETARG_Ax(*pc) * (MAXARG_C + 1); @@ -1810,12 +1869,14 @@ void luaV_execute (lua_State *L, CallInfo *ci) { vmbreak; } vmcase(OP_CLOSURE) { + StkId ra = RA(i); Proto *p = cl->p->p[GETARG_Bx(i)]; halfProtect(pushclosure(L, p, cl->upvals, base, ra)); checkGC(L, ra + 1); vmbreak; } vmcase(OP_VARARG) { + StkId ra = RA(i); int n = GETARG_C(i) - 1; /* required results */ Protect(luaT_getvarargs(L, ci, ra, n)); vmbreak; diff --git a/src/lua/lvm.h b/src/lua/lvm.h index 1bc16f3a5..dba1ad277 100644 --- a/src/lua/lvm.h +++ b/src/lua/lvm.h @@ -110,6 +110,11 @@ typedef enum { luaC_barrierback(L, gcvalue(t), v); } +/* +** Shift right is the same as shift left with a negative 'y' +*/ +#define luaV_shiftr(x,y) luaV_shiftl(x,intop(-, 0, y)) + LUAI_FUNC int luaV_equalobj (lua_State *L, const TValue *t1, const TValue *t2); From 866ca4b657d30236307c2d5275da35ff0fef83c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Apr 2023 21:11:51 +0200 Subject: [PATCH 26/78] Apply Pi-hole Lua patches Signed-off-by: DL6ER --- patch/lua/0001-add-pihole-library.patch | 4 ++-- src/lua/linit.c | 3 +++ src/lua/lua.c | 18 ++++++++++++++++-- src/lua/luac.c | 4 +++- src/lua/lualib.h | 4 ++++ 5 files changed, 28 insertions(+), 5 deletions(-) diff --git a/patch/lua/0001-add-pihole-library.patch b/patch/lua/0001-add-pihole-library.patch index 30a0f592a..dc3885aa5 100644 --- a/patch/lua/0001-add-pihole-library.patch +++ b/patch/lua/0001-add-pihole-library.patch @@ -48,9 +48,9 @@ index 454ce12f..a363925c 100644 + ftl_lua_init(L); + /***************************************************/ + - if (!runargs(L, argv, script)) /* execute arguments -e and -l */ + if (!runargs(L, argv, optlim)) /* execute arguments -e and -l */ return 0; /* something failed */ - if (script < argc && /* execute main script (if there is one) */ + if (script > 0) { /* execute main script (if there is one) */ @@ -616,7 +622,9 @@ static int pmain (lua_State *L) { } diff --git a/src/lua/linit.c b/src/lua/linit.c index 69808f84f..9a5bcfdcc 100644 --- a/src/lua/linit.c +++ b/src/lua/linit.c @@ -50,6 +50,9 @@ static const luaL_Reg loadedlibs[] = { {LUA_MATHLIBNAME, luaopen_math}, {LUA_UTF8LIBNAME, luaopen_utf8}, {LUA_DBLIBNAME, luaopen_debug}, + /****** Pi-hole modification ******/ + {LUA_PIHOLELIBNAME, luaopen_pihole}, + /**********************************/ {NULL, NULL} }; diff --git a/src/lua/lua.c b/src/lua/lua.c index 0ff884545..f269c9973 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -20,6 +20,10 @@ #include "lauxlib.h" #include "lualib.h" +/** Pi-hole modification **/ +#include "ftl_lua.h" +/**************************/ + #if !defined(LUA_PROGNAME) #define LUA_PROGNAME "lua" @@ -211,7 +215,9 @@ static int dostring (lua_State *L, const char *s, const char *name) { /* ** Receives 'globname[=modname]' and runs 'globname = require(modname)'. */ -static int dolibrary (lua_State *L, char *globname) { +/************** Pi-hole modification ***************/ +int dolibrary (lua_State *L, char *globname) { +/***************************************************/ int status; char *modname = strchr(globname, '='); if (modname == NULL) /* no explicit name? */ @@ -639,6 +645,12 @@ static int pmain (lua_State *L) { if (handle_luainit(L) != LUA_OK) /* run LUA_INIT */ return 0; /* error running LUA_INIT */ } + + /************** Pi-hole modification ***************/ + // Load and enable libraries bundled with Pi-hole + ftl_lua_init(L); + /***************************************************/ + if (!runargs(L, argv, optlim)) /* execute arguments -e and -l */ return 0; /* something failed */ if (script > 0) { /* execute main script (if there is one) */ @@ -659,7 +671,9 @@ static int pmain (lua_State *L) { } -int main (int argc, char **argv) { +/******* Pi-hole modification ********/ +int lua_main (int argc, char **argv) { +/*************************************/ int status, result; lua_State *L = luaL_newstate(); /* create state */ if (L == NULL) { diff --git a/src/lua/luac.c b/src/lua/luac.c index 5f4a141bd..139468cc6 100644 --- a/src/lua/luac.c +++ b/src/lua/luac.c @@ -193,7 +193,9 @@ static int pmain(lua_State* L) return 0; } -int main(int argc, char* argv[]) +/******* Pi-hole modification ********/ +int luac_main(int argc, char* argv[]) +/*************************************/ { lua_State* L; int i=doargs(argc,argv); diff --git a/src/lua/lualib.h b/src/lua/lualib.h index 262552907..a7cab4eb0 100644 --- a/src/lua/lualib.h +++ b/src/lua/lualib.h @@ -44,6 +44,10 @@ LUAMOD_API int (luaopen_debug) (lua_State *L); #define LUA_LOADLIBNAME "package" LUAMOD_API int (luaopen_package) (lua_State *L); +/************ Pi-hole modification *************/ +#define LUA_PIHOLELIBNAME "pihole" +LUAMOD_API int (luaopen_pihole) (lua_State *L); +/***********************************************/ /* open all previous libraries */ LUALIB_API void (luaL_openlibs) (lua_State *L); From 49bc744846c1529e2dde9cebb7d343bd179bf981 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 3 May 2023 21:53:30 +0200 Subject: [PATCH 27/78] Add header analysis also in tcp_key_recurse to fix an issue with wrong upstream servers being attributed to DNSSEC-related queries when multiple upstream servers are defined (e.g. conditional forwarding) Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 1f2eb2ddf..389a07551 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2089,6 +2089,8 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si unsigned char *packet = NULL; struct dns_header *new_header = NULL; + FTL_header_analysis(header->hb4, 0, server, daemon->log_display_id); + while (1) { size_t m; From 7871ebe32bf4e873a0acef7981af6f23f26c88fe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 4 May 2023 07:21:34 +0200 Subject: [PATCH 28/78] Add extra debugging output Signed-off-by: DL6ER --- src/config.c | 8 -------- src/dnsmasq/forward.c | 2 +- src/dnsmasq_interface.c | 7 ++++++- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/config.c b/src/config.c index eae5cdb8a..a00c31df0 100644 --- a/src/config.c +++ b/src/config.c @@ -837,9 +837,6 @@ static char *parse_FTLconf(FILE *fp, const char *key) // Go to beginning of file fseek(fp, 0L, SEEK_SET); - if(config.debug & DEBUG_EXTRA) - logg("initial: conflinebuffer = %p, keystr = %p, size = %zu", conflinebuffer, keystr, size); - // Set size to zero if conflinebuffer is not available here // This causes getline() to allocate memory for the buffer itself if(conflinebuffer == NULL && size != 0) @@ -848,11 +845,6 @@ static char *parse_FTLconf(FILE *fp, const char *key) errno = 0; while(getline(&conflinebuffer, &size, fp) != -1) { - if(config.debug & DEBUG_EXTRA) - { - logg("conflinebuffer = %p, keystr = %p, size = %zu", conflinebuffer, keystr, size); - logg(" while reading line \"%s\" looking for \"%s\"", conflinebuffer, keystr); - } // Check if memory allocation failed if(conflinebuffer == NULL) break; diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index 389a07551..ad01c1b94 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -2089,7 +2089,7 @@ static int tcp_key_recurse(time_t now, int status, struct dns_header *header, si unsigned char *packet = NULL; struct dns_header *new_header = NULL; - FTL_header_analysis(header->hb4, 0, server, daemon->log_display_id); + FTL_header_analysis(header->hb4, RCODE(header), server, daemon->log_display_id); while (1) { diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index e18b48309..e73e745e3 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -2614,7 +2614,12 @@ void _FTL_header_analysis(const unsigned char header4, const unsigned int rcode, { memcpy(&last_server, &server->addr, sizeof(last_server)); if(config.debug & DEBUG_EXTRA) - logg("Got forward address: YES"); + { + char ip[ADDRSTRLEN+1] = { 0 }; + in_port_t port = 0; + mysockaddr_extract_ip_port(&last_server, ip, &port); + logg("Got forward address: %s#%u (%s:%i)", ip, port, short_path(file), line); + } } else { From a2746d70c51cd9958cac5b8fb13e64d9b8a83b64 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 4 May 2023 11:32:03 +0200 Subject: [PATCH 29/78] Also analyze UDP reply headers Signed-off-by: DL6ER --- src/dnsmasq/forward.c | 2 ++ src/dnsmasq_interface.c | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/src/dnsmasq/forward.c b/src/dnsmasq/forward.c index ad01c1b94..60e267640 100644 --- a/src/dnsmasq/forward.c +++ b/src/dnsmasq/forward.c @@ -1170,6 +1170,8 @@ void reply_query(int fd, time_t now) server = daemon->serverarray[c]; + FTL_header_analysis(header->hb4, RCODE(header), server, daemon->log_display_id); + if (RCODE(header) != REFUSED) daemon->serverarray[first]->last_server = c; else if (daemon->serverarray[first]->last_server == c) diff --git a/src/dnsmasq_interface.c b/src/dnsmasq_interface.c index e73e745e3..3a4be64a5 100644 --- a/src/dnsmasq_interface.c +++ b/src/dnsmasq_interface.c @@ -1877,6 +1877,8 @@ static void update_upstream(queriesData *query, const int id) id, oldaddr, oldport, ip, port); } } + + // Update upstream server ID query->upstreamID = upstreamID; } } @@ -2025,6 +2027,10 @@ static void FTL_reply(const unsigned int flags, const char *name, const union al if(!cached) update_upstream(query, id); + // Reset last_server to avoid possibly changing the upstream server + // again in the next query + memset(&last_server, 0, sizeof(last_server)); + // Save response time // Skipped internally if already computed set_response_time(query, response); @@ -2530,6 +2536,9 @@ static void FTL_upstream_error(const union all_addr *addr, const unsigned int fl // Set query reply query_set_reply(0, reply, addr, query, response); + // Reset last_server + memset(&last_server, 0, sizeof(last_server)); + // Unlock shared memory unlock_shm(); } From 51ca8f369b49df0051b5d41649448f9747defb41 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 May 2023 22:11:35 +0200 Subject: [PATCH 30/78] Do not log running out of disk space when the disk occupation is > 100%. We are seeing this with docker deployments on macOS hosts. It is a band-aid fix, however, it also seem to be the only thing we can do given that docker didn't fix this in nearly two years now. Signed-off-by: DL6ER --- src/gc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gc.c b/src/gc.c index 47734c110..0fbfd359d 100644 --- a/src/gc.c +++ b/src/gc.c @@ -86,9 +86,9 @@ static int check_space(const char *file, int LastUsage) // exceeds the configured threshold and current usage is higher than // usage in the last run (to prevent log spam) perc = get_filepath_usage(file, buffer); - if(perc > config.check.disk && perc > LastUsage ) + if(perc > config.check.disk && perc > LastUsage && perc <= 100.0) log_resource_shortage(-1.0, 0, -1, perc, file, buffer); - + return perc; } From b05ada45e9b9b0cc16219ee20cc7cea539e570aa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 13 May 2023 12:40:39 +0200 Subject: [PATCH 31/78] Move dhcp-discover into a dedicated "tools" target Signed-off-by: DL6ER --- src/CMakeLists.txt | 4 ++-- src/args.c | 2 +- src/tools/CMakeLists.txt | 18 ++++++++++++++++++ src/{ => tools}/dhcp-discover.c | 0 src/{ => tools}/dhcp-discover.h | 0 5 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 src/tools/CMakeLists.txt rename src/{ => tools}/dhcp-discover.c (100%) rename src/{ => tools}/dhcp-discover.h (100%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e71381793..cee7b1b32 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -116,8 +116,6 @@ set(sources daemon.h datastructure.c datastructure.h - dhcp-discover.c - dhcp-discover.h dnsmasq_interface.c dnsmasq_interface.h edns0.c @@ -180,6 +178,7 @@ add_executable(pihole-FTL $ $ $ + $ ) if(STATIC STREQUAL "true") set_target_properties(pihole-FTL PROPERTIES LINK_SEARCH_START_STATIC ON) @@ -253,3 +252,4 @@ add_subdirectory(lua) add_subdirectory(lua/scripts) add_subdirectory(tre-regex) add_subdirectory(syscalls) +add_subdirectory(tools) diff --git a/src/args.c b/src/args.c index e5917f154..2df29b46e 100644 --- a/src/args.c +++ b/src/args.c @@ -33,7 +33,7 @@ // LUA dependencies #include "lua/ftl_lua.h" // run_dhcp_discover() -#include "dhcp-discover.h" +#include "tools/dhcp-discover.h" // defined in dnsmasq.c extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal); diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt new file mode 100644 index 000000000..d13a5942e --- /dev/null +++ b/src/tools/CMakeLists.txt @@ -0,0 +1,18 @@ +# Pi-hole: A black hole for Internet advertisements +# (c) 2020 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# FTL Engine +# /src/tools/CMakeList.txt +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +set(tools_sources + dhcp-discover.c + dhcp-discover.h + ) + +add_library(tools OBJECT ${tools_sources}) +target_compile_options(tools PRIVATE "${EXTRAWARN}") +target_include_directories(tools PRIVATE ${PROJECT_SOURCE_DIR}/src) diff --git a/src/dhcp-discover.c b/src/tools/dhcp-discover.c similarity index 100% rename from src/dhcp-discover.c rename to src/tools/dhcp-discover.c diff --git a/src/dhcp-discover.h b/src/tools/dhcp-discover.h similarity index 100% rename from src/dhcp-discover.h rename to src/tools/dhcp-discover.h From 29a33102f3b63272f86f54e49ce9957eeca973ad Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 13 May 2023 20:51:02 +0200 Subject: [PATCH 32/78] Add pihole-FTL arp-scan [{-v,-a}] Signed-off-by: DL6ER --- src/args.c | 37 ++- src/syscalls/recvfrom.c | 11 +- src/syscalls/sendto.c | 4 +- src/tools/CMakeLists.txt | 2 + src/tools/arp-scan.c | 486 ++++++++++++++++++++++++++++++++++++++ src/tools/arp-scan.h | 16 ++ src/tools/dhcp-discover.c | 2 +- src/tools/dhcp-discover.h | 1 + 8 files changed, 542 insertions(+), 17 deletions(-) create mode 100644 src/tools/arp-scan.c create mode 100644 src/tools/arp-scan.h diff --git a/src/args.c b/src/args.c index 2df29b46e..f86a19e30 100644 --- a/src/args.c +++ b/src/args.c @@ -34,6 +34,8 @@ #include "lua/ftl_lua.h" // run_dhcp_discover() #include "tools/dhcp-discover.h" +// run_arp_scan() +#include "tools/arp-scan.h" // defined in dnsmasq.c extern void print_dnsmasq_version(const char *yellow, const char *green, const char *bold, const char *normal); @@ -163,6 +165,24 @@ void parse_args(int argc, char* argv[]) (argc > 1 && strEndsWith(argv[1], ".db"))) exit(sqlite3_shell_main(argc, argv)); + // DHCP discovery mode + if(argc > 1 && strcmp(argv[1], "dhcp-discover") == 0) + { + // Enable stdout printing + cli_mode = true; + exit(run_dhcp_discover()); + } + + // ARP scanning mode + if(argc > 1 && strcmp(argv[1], "arp-scan") == 0) + { + // Enable stdout printing + cli_mode = true; + const bool verbose = argc > 2 && strcmp(argv[2], "-v") == 0; + const bool arp_all = argc > 2 && strcmp(argv[2], "-a") == 0; + exit(run_arp_scan(verbose, arp_all)); + } + // start from 1, as argv[0] is the executable name for(int i = 1; i < argc; i++) { @@ -415,14 +435,6 @@ void parse_args(int argc, char* argv[]) } } - // Regex test mode - if(strcmp(argv[i], "dhcp-discover") == 0) - { - // Enable stdout printing - cli_mode = true; - exit(run_dhcp_discover()); - } - // List of implemented arguments if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "help") == 0 || strcmp(argv[i], "--help") == 0) { @@ -495,13 +507,18 @@ void parse_args(int argc, char* argv[]) printf("%sDebugging and special use:%s\n", yellow, normal); printf("\t%sd%s, %sdebug%s Enter debugging mode\n", green, normal, green, normal); - printf("\t%stest%s Don't start pihole-FTL but\n", green, normal); - printf("\t instead quit immediately\n"); + printf("\t%stest%s Don't start pihole-FTL but instead\n", green, normal); + printf("\t quit immediately\n"); printf("\t%s-f%s, %sno-daemon%s Don't go into daemon mode\n\n", green, normal, green, normal); printf("%sOther:%s\n", yellow, normal); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); + printf("\t%sarp-scan %s[{-v/-a}]%s Use ARP to scan local network for\n", green, cyan, normal); + printf("\t possible IP conflicts\n"); + printf("\t Append %s-v%s for verbose output mode\n", cyan, normal); + printf("\t Append %s-a%s to force scan on all\n", cyan, normal); + printf("\t interfaces\n"); printf("\t%s-h%s, %shelp%s Display this help and exit\n\n", green, normal, green, normal); exit(EXIT_SUCCESS); } diff --git a/src/syscalls/recvfrom.c b/src/syscalls/recvfrom.c index 8db0ac468..4dff79285 100644 --- a/src/syscalls/recvfrom.c +++ b/src/syscalls/recvfrom.c @@ -32,11 +32,14 @@ ssize_t FTLrecvfrom(int sockfd, void *buf, size_t len, int flags, struct sockadd // Backup errno value const int _errno = errno; - // Final error checking (may have failed for some other reason then an - // EINTR = interrupted system call) - if(ret < 0) + // Final error checking. May have failed for some other reason then an + // EINTR = interrupted system call. In that case, log a warning However, + // if the error is EAGAIN, this is not an error, but just a non-blocking + // socket that has no data available or we ran into an (expected) + // timeout. In that case, do not log a warning + if(ret < 0 && errno != EAGAIN) logg("WARN: Could not recvfrom() in %s() (%s:%i): %s", - func, file, line, strerror(errno)); + func, file, line, strerror(errno)); // Restore errno value errno = _errno; diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c index cc7234c94..846c571e7 100644 --- a/src/syscalls/sendto.c +++ b/src/syscalls/sendto.c @@ -33,8 +33,8 @@ ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct soc const int _errno = errno; // Final error checking (may have failed for some other reason then an - // EINTR = interrupted system call) - if(ret < 0) + // EINTR = interrupted system call), also ignore EPROTONOSUPPORT (ARP scanning) + if(ret < 0 && errno != EPROTONOSUPPORT) logg("WARN: Could not sendto() in %s() (%s:%i): %s", func, file, line, strerror(errno)); diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index d13a5942e..47de88b1a 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -9,6 +9,8 @@ # Please see LICENSE file for your rights under this license. set(tools_sources + arp-scan.c + arp-scan.h dhcp-discover.c dhcp-discover.h ) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c new file mode 100644 index 000000000..63bbdc295 --- /dev/null +++ b/src/tools/arp-scan.c @@ -0,0 +1,486 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2023 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* ARP scanning routines +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ +// Inspired by https://stackoverflow.com/a/39287433 but heavily modified + +#include "FTL.h" +#include "arp-scan.h" +#include "log.h" +// get_hardware_address() +#include "dhcp-discover.h" + +#include +#include +#include +//htons etc +#include + +// How many threads do we spawn at maximum? +// This is also the limit for interfaces +// we scan for DHCP activity. +#define MAXTHREADS 32 +#define MAX_MACS 3 +#define NUM_SCANS 10 +#define ARP_TIMEOUT 1 + +// Global lock used by all threads +static pthread_mutex_t lock; +static bool arp_verbose = false; +static bool arp_all = false; + +#define PROTO_ARP 0x0806 +#define ETH2_HEADER_LEN 14 +#define HW_TYPE 1 +#define MAC_LENGTH 6 +#define IPV4_LENGTH 4 +#define ARP_REQUEST 0x01 +#define ARP_REPLY 0x02 +#define BUF_SIZE 60 + +#pragma pack(push, 1) + +// ARP header struct +// See https://en.wikipedia.org/wiki/Address_Resolution_Protocol#Packet_structure +struct arp_header { + unsigned short hardware_type; + unsigned short protocol_type; + unsigned char hardware_len; + unsigned char protocol_len; + unsigned short opcode; + unsigned char sender_mac[MAC_LENGTH]; + unsigned char sender_ip[IPV4_LENGTH]; + unsigned char target_mac[MAC_LENGTH]; + unsigned char target_ip[IPV4_LENGTH]; +}; +#pragma pack(pop) + +struct arp_result { + unsigned int replied[NUM_SCANS]; + unsigned char mac[MAX_MACS][MAC_LENGTH]; +}; + +// Sends multiple ARP who-has request on interface ifindex, using source mac src_mac and source ip src_ip. +// Interates over all IP addresses in the range of dst_ip/cidr. +static int send_arps(const int fd, const int ifindex, const char *iface, const unsigned char *src_mac, + struct in_addr *src_ip, struct in_addr dst_ip, const int dst_cidr) +{ + int err = -1; + unsigned char buffer[BUF_SIZE]; + memset(buffer, 0, sizeof(buffer)); + + // Construct the Ethernet header + struct sockaddr_ll socket_address; + socket_address.sll_family = AF_PACKET; + socket_address.sll_protocol = htons(ETH_P_ARP); + socket_address.sll_ifindex = ifindex; + socket_address.sll_hatype = htons(ARPHRD_ETHER); + socket_address.sll_pkttype = PACKET_BROADCAST; + socket_address.sll_halen = MAC_LENGTH; + socket_address.sll_addr[6] = 0; + socket_address.sll_addr[7] = 0; + + struct ethhdr *send_req = (struct ethhdr *) buffer; + struct arp_header *arp_req = (struct arp_header *) (buffer + ETH2_HEADER_LEN); + ssize_t ret; + + // Destination is the broadcast address + memset(send_req->h_dest, 0xff, MAC_LENGTH); + + // Target MAC is zero (we don't know it) + memset(arp_req->target_mac, 0x00, MAC_LENGTH); + + // Source MAC to our own MAC address + memcpy(send_req->h_source, src_mac, MAC_LENGTH); + memcpy(arp_req->sender_mac, src_mac, MAC_LENGTH); + memcpy(socket_address.sll_addr, src_mac, MAC_LENGTH); + + // Protocol type is ARP + send_req->h_proto = htons(ETH_P_ARP); + + // Create ARP request + arp_req->hardware_type = htons(HW_TYPE); + arp_req->protocol_type = htons(ETH_P_IP); + arp_req->hardware_len = MAC_LENGTH; + arp_req->protocol_len = IPV4_LENGTH; + arp_req->opcode = htons(ARP_REQUEST); + + // Copy IP address to arp_req + memcpy(arp_req->sender_ip, &src_ip->s_addr, sizeof(src_ip->s_addr)); + + // Loop over all possible IP addresses in the range dst_ip/cidr + // We start at 1 because the first IP address has already been set above + for(unsigned int i = 0; i < (1u << (32 - dst_cidr)); i++) + { + // Fill in target IP address + memcpy(arp_req->target_ip, &dst_ip.s_addr, sizeof(dst_ip.s_addr)); + +#ifdef DEBUG + printf("Sending ARP request for %s@%s\n", inet_ntoa(*dst_ip), iface); +#endif + + // Send ARP request + ret = sendto(fd, buffer, 42, 0, (struct sockaddr *) &socket_address, sizeof(socket_address)); + if (ret == -1) + { + if(errno != EPROTONOSUPPORT) + printf("Unable to send ARP request for %s@%s: %s\n", + inet_ntoa(dst_ip), iface, strerror(errno)); + goto out; + } + + // Increment IP address + dst_ip.s_addr = htonl(ntohl(dst_ip.s_addr) + 1); + } + + err = 0; +out: + return err; +} + +static int create_arp_socket(const int ifindex, const char *iface) +{ + // Create socket for ARP communications + const int arp_socket = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ARP)); + if(arp_socket < 0) + { + printf("Unable to create socket for ARP communications on interface %s: %s\n", iface, strerror(errno)); + return -1; + } + + // Bind socket to interface + struct sockaddr_ll sll; + memset(&sll, 0, sizeof(struct sockaddr_ll)); + sll.sll_family = AF_PACKET; + sll.sll_ifindex = ifindex; + if (bind(arp_socket, (struct sockaddr*) &sll, sizeof(struct sockaddr_ll)) < 0) + { + printf("Unable to bind socket for ARP communications on interface %s: %s\n", iface, strerror(errno)); + close(arp_socket); + return -1; + } + + // Set timeout + struct timeval tv; + tv.tv_sec = ARP_TIMEOUT; + tv.tv_usec = 0; + if (setsockopt(arp_socket, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0) + { + printf("Unable to set timeout for ARP communications on interface %s: %s\n", iface, strerror(errno)); + close(arp_socket); + return -1; + } + + return arp_socket; +} + +// Read all ARP responses +static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, + struct arp_result *result, const size_t result_len, const unsigned int scan_id) +{ + ssize_t ret = 0; + unsigned char buffer[BUF_SIZE]; + + // Read ARP responses + while(ret >= 0) + { + ret = recvfrom(fd, buffer, BUF_SIZE, 0, NULL, NULL); + if (ret == -1) + { + if(errno == EAGAIN) + { + // Timeout + ret = 0; + break; + } + + // Error + printf("recvfrom(): %s", strerror(errno)); + break; + } + struct ethhdr *rcv_resp = (struct ethhdr *) buffer; + struct arp_header *arp_resp = (struct arp_header *) (buffer + ETH2_HEADER_LEN); + if (ntohs(rcv_resp->h_proto) != PROTO_ARP) + { +#ifdef DEBUG + printf("Not an ARP packet"); +#endif + continue; + } + if (ntohs(arp_resp->opcode) != ARP_REPLY) + { +#ifdef DEBUG + printf("Not an ARP reply"); +#endif + continue; + } +#ifdef DEBUG + printf("received ARP len=%ld", ret); +#endif + struct in_addr sender_a; + memcpy(&sender_a.s_addr, arp_resp->sender_ip, sizeof(sender_a.s_addr)); + +#ifdef DEBUG + printf("%-16s %-20s\t%02x:%02x:%02x:%02x:%02x:%02x", + iface, inet_ntoa(sender_a), + arp_resp->sender_mac[0], + arp_resp->sender_mac[1], + arp_resp->sender_mac[2], + arp_resp->sender_mac[3], + arp_resp->sender_mac[4], + arp_resp->sender_mac[5]); +#endif + + // Check if we have already found this IP address + uint32_t i = ntohl(sender_a.s_addr) - ntohl(dst_ip->s_addr); + if(i >= result_len) + { + printf("Received IP address %s out of range\n", inet_ntoa(sender_a)); + continue; + } + + // Memorize that we have received a reply for this IP address + result[i].replied[scan_id]++; + + // Save MAC address + for(unsigned int j = 0; j < MAX_MACS; j++) + { + // Check if received MAC is already stored in result[i].mac[j] + if(memcmp(result[i].mac[j], arp_resp->sender_mac, MAC_LENGTH) == 0) + { + break; + } + // Check if result[i].mac[j] is all-zero + if(memcmp(result[i].mac[j], "\x00\x00\x00\x00\x00\x00", MAC_LENGTH) == 0) + { + // Copy MAC address to result[i].mac[j] + memcpy(result[i].mac[j], arp_resp->sender_mac, sizeof(arp_resp->sender_mac)); + break; + } + } + } + + return ret; +} + +// Convert netmask to CIDR +static int netmask_to_cidr(struct in_addr *addr) +{ + // Count the number of set bits in an unsigned integer + return __builtin_popcount(addr->s_addr); +} + +static void *arp_scan_iface(void *args) +{ + // Get interface details + struct ifaddrs *ifa = (struct ifaddrs*)args; + + // Get interface name + const char *iface = ifa->ifa_name; + + // Set interface name as thread name + prctl(PR_SET_NAME, iface, 0, 0, 0); + + // Get interface IPv4 address + struct sockaddr_in src_addr = { 0 }; + memcpy(&src_addr, ((struct ifaddrs*)args)->ifa_addr, sizeof(src_addr)); + char ipstr[INET_ADDRSTRLEN] = { 0 }; + inet_ntop(AF_INET, &src_addr.sin_addr, ipstr, INET_ADDRSTRLEN); + + // Get interface netmask + struct sockaddr_in mask = { 0 }; + memcpy(&mask, ((struct ifaddrs*)args)->ifa_netmask, sizeof(mask)); + // char netmask[INET_ADDRSTRLEN] = { 0 }; + // inet_ntop(AF_INET, &mask.sin_addr, netmask, INET_ADDRSTRLEN); + + // Convert subnet to CIDR + const int cidr = netmask_to_cidr(&mask.sin_addr); + + // Get interface index + const int ifindex = if_nametoindex(iface); + + // Scan only interfaces with CIDR >= 24 + if(cidr < 24 && !arp_all) + { + printf("Skipped interface %s (%s/%i)\n", iface, ipstr, cidr); + pthread_exit(NULL); + } + if(arp_verbose) + printf("Scanning interface %s (%s/%i)...\n", iface, ipstr, cidr); + + // Create socket for ARP communications + const int arp_socket = create_arp_socket(ifindex, iface); + + // Cannot create socket, likely a permission error + if(arp_socket < 0) + pthread_exit(NULL); + + // Get hardware address of client machine + unsigned char mac[16] = { 0 }; + get_hardware_address(arp_socket, iface, mac); + + // Define destination IP address by masking source IP with netmask + struct in_addr dst_addr = { 0 }; + dst_addr.s_addr = src_addr.sin_addr.s_addr & mask.sin_addr.s_addr; + + // Allocate memory for ARP response buffer + const size_t arp_result_len = 1 << (32 - cidr); + struct arp_result *result = calloc(arp_result_len, sizeof(struct arp_result)); + + for(unsigned int scan_id = 0; scan_id < NUM_SCANS; scan_id++) + { +#ifdef DEBUG + printf("Scanning interface %s (%s/%i) for the %i. time\n", iface, ipstr, cidr, scan_id + 1); +#endif + // Send ARP requests to all IPs in subnet + if(send_arps(arp_socket, ifindex, iface, mac, &src_addr.sin_addr, dst_addr, cidr) != 0) + break; + + // Read ARP responses + if(read_arp(arp_socket, iface, &dst_addr, result, arp_result_len, scan_id) != 0) + break; + } + + // Check if there are any results + unsigned int replies = 0; + for(unsigned int i = 0; i < arp_result_len; i++) + for(unsigned int j = 0; j < NUM_SCANS; j++) + replies += result[i].replied[j]; + + if(pthread_mutex_lock(&lock) != 0) + return NULL; + + if(replies == 0) + { + printf("No devices found on interface %s (%s/%i)\n", iface, ipstr, cidr); + goto arp_scan_iface_end; + } + + // Print results + printf("ARP scan on interface %s (%s/%i) finished\n", iface, ipstr, cidr); + printf("%-20s %-16s %-17s Reply matrix\n", "IP address", "Interface", "MAC address"); + for(unsigned int i = 0; i < arp_result_len; i++) + { + // Check if IP address replied + bool replied = false, multiple_replies = false; + for(unsigned int j = 0; j < NUM_SCANS; j++) + { + if(result[i].replied[j] > 0) + { + replied = true; + multiple_replies |= result[i].replied[j] > 1; + } + } + if(!replied) + continue; + + // Convert IP address to string + struct in_addr ip = { 0 }; + ip.s_addr = htonl(ntohl(dst_addr.s_addr) + i); + inet_ntop(AF_INET, &ip, ipstr, INET_ADDRSTRLEN); + + // Print MAC addresses + unsigned int j = 0; + for(j = 0; j < MAX_MACS; j++) + { + // Check if result[i].mac[j] is all-zero + if(memcmp(result[i].mac[j], "\x00\x00\x00\x00\x00\x00", 6) == 0) + break; + + // Print MAC address + printf("%-20s %-16s %02x:%02x:%02x:%02x:%02x:%02x ", + ipstr, iface, + result[i].mac[j][0], + result[i].mac[j][1], + result[i].mac[j][2], + result[i].mac[j][3], + result[i].mac[j][4], + result[i].mac[j][5]); + + for(unsigned int k = 0; k < NUM_SCANS; k++) + { + printf(" %s", result[i].replied[k] > 0 ? "X" : "-"); + } + putc('\n', stdout); + } + + // Print warning if multiple MAC addresses replied + if(j > 1) + printf("WARNING: Multiple MAC addresses replied as %s\n", ipstr); + if(multiple_replies) + printf("WARNING: Received multiple replies for %s\n", ipstr); + } + putc('\n', stdout); + +arp_scan_iface_end: + if(pthread_mutex_unlock(&lock) != 0) + return NULL; + + // Close socket + close(arp_socket); + pthread_exit(NULL); +} + +int run_arp_scan(const bool verbose, const bool scan_all) +{ + arp_verbose = verbose; + arp_all = scan_all; + puts("Discovering IPv4 hosts on the network using the Address Resolution Protocol (ARP)...\n"); + + // Get interface names for available interfaces on this machine + // and launch a thread for each one + pthread_t scanthread[MAXTHREADS]; + pthread_attr_t attr; + // Initialize thread attributes object with default attribute values + pthread_attr_init(&attr); + + // Create processing/logging lock + pthread_mutexattr_t lock_attr = {}; + // Initialize the lock attributes + pthread_mutexattr_init(&lock_attr); + // Initialize the lock + pthread_mutex_init(&lock, &lock_attr); + // Destroy the lock attributes since we're done with it + pthread_mutexattr_destroy(&lock_attr); + + struct ifaddrs *addrs, *tmp; + getifaddrs(&addrs); + tmp = addrs; + + // Loop until there are no more interfaces available + // or we reached the maximum number of threads + int tid = 0; + while(tmp != NULL && tid < MAXTHREADS) + { + // Create a thread for interfaces of type AF_INET + if(tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_INET) + { + if(pthread_create(&scanthread[tid], &attr, arp_scan_iface, tmp ) != 0) + { + printf("Unable to launch thread for interface %s, skipping...\n", + tmp->ifa_name); + continue; + } + + // Increase thread ID + tid++; + } + + // Advance to the next interface + tmp = tmp->ifa_next; + } + + // Wait for all threads to join back with us + for(tid--; tid > -1; tid--) + pthread_join(scanthread[tid], NULL); + + // Free linked-list of interfaces on this client + freeifaddrs(addrs); + + return EXIT_SUCCESS; +} diff --git a/src/tools/arp-scan.h b/src/tools/arp-scan.h new file mode 100644 index 000000000..90b5fec70 --- /dev/null +++ b/src/tools/arp-scan.h @@ -0,0 +1,16 @@ +/* Pi-hole: A black hole for Internet advertisements +* (c) 2023 Pi-hole, LLC (https://pi-hole.net) +* Network-wide ad blocking via your own hardware. +* +* FTL Engine +* ARP scanning prototypes +* +* This file is copyright under the latest version of the EUPL. +* Please see LICENSE file for your rights under this license. */ + +#ifndef ARP_SCAN_H +#define ARP_SCAN_H + +int run_arp_scan(const bool verbose, const bool scan_all); + +#endif // ARP_SCAN_H diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 547135525..5e2613eaa 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -128,7 +128,7 @@ static int create_dhcp_socket(const char *iname) } // determines hardware address on client machine -static int get_hardware_address(const int sock, const char *iname, unsigned char *mac) +int get_hardware_address(const int sock, const char *iname, unsigned char *mac) { struct ifreq ifr; strncpy((char *)&ifr.ifr_name, iname, sizeof(ifr.ifr_name)-1); diff --git a/src/tools/dhcp-discover.h b/src/tools/dhcp-discover.h index 7d9bc7e84..efd07c116 100644 --- a/src/tools/dhcp-discover.h +++ b/src/tools/dhcp-discover.h @@ -12,5 +12,6 @@ #define DHCP_DISCOVER_H int run_dhcp_discover(void); +int get_hardware_address(const int sock, const char *iname, unsigned char *mac); #endif // DHCP_DISCOVER_H From 36bd2a56e4e804c82332caca8d30f354a1106fd5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 13 May 2023 22:22:17 +0200 Subject: [PATCH 33/78] Unify warning Signed-off-by: DL6ER --- src/tools/arp-scan.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 63bbdc295..cf47cf6cf 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -66,7 +66,7 @@ struct arp_result { }; // Sends multiple ARP who-has request on interface ifindex, using source mac src_mac and source ip src_ip. -// Interates over all IP addresses in the range of dst_ip/cidr. +// Iterates over all IP addresses in the range of dst_ip/cidr. static int send_arps(const int fd, const int ifindex, const char *iface, const unsigned char *src_mac, struct in_addr *src_ip, struct in_addr dst_ip, const int dst_cidr) { @@ -409,10 +409,8 @@ static void *arp_scan_iface(void *args) putc('\n', stdout); } - // Print warning if multiple MAC addresses replied - if(j > 1) - printf("WARNING: Multiple MAC addresses replied as %s\n", ipstr); - if(multiple_replies) + // Print warning if we received multiple replies + if(j > 1 || multiple_replies) printf("WARNING: Received multiple replies for %s\n", ipstr); } putc('\n', stdout); From 26396bd20b9a77b788e695b874c93172b4ec35e1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 May 2023 07:26:06 +0200 Subject: [PATCH 34/78] Use dedicated counters per MAC for a more accurate per-device reply matrix Signed-off-by: DL6ER --- src/tools/arp-scan.c | 86 +++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index cf47cf6cf..09aace8c4 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -61,8 +61,10 @@ struct arp_header { #pragma pack(pop) struct arp_result { - unsigned int replied[NUM_SCANS]; - unsigned char mac[MAX_MACS][MAC_LENGTH]; + struct device { + unsigned int replied[NUM_SCANS]; + unsigned char mac[MAC_LENGTH]; + } device[MAX_MACS]; }; // Sends multiple ARP who-has request on interface ifindex, using source mac src_mac and source ip src_ip. @@ -244,30 +246,32 @@ static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, continue; } - // Memorize that we have received a reply for this IP address - result[i].replied[scan_id]++; - // Save MAC address - for(unsigned int j = 0; j < MAX_MACS; j++) + unsigned int j = 0; + for(; j < MAX_MACS; j++) { - // Check if received MAC is already stored in result[i].mac[j] - if(memcmp(result[i].mac[j], arp_resp->sender_mac, MAC_LENGTH) == 0) + // Check if received MAC is already stored in result[i].device[j].mac + if(memcmp(result[i].device[j].mac, arp_resp->sender_mac, MAC_LENGTH) == 0) { break; } - // Check if result[i].mac[j] is all-zero - if(memcmp(result[i].mac[j], "\x00\x00\x00\x00\x00\x00", MAC_LENGTH) == 0) + // Check if result[i].device[j].mac is all-zero + if(memcmp(result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", MAC_LENGTH) == 0) { - // Copy MAC address to result[i].mac[j] - memcpy(result[i].mac[j], arp_resp->sender_mac, sizeof(arp_resp->sender_mac)); + // Copy MAC address to result[i].device[j].mac + memcpy(result[i].device[j].mac, arp_resp->sender_mac, sizeof(arp_resp->sender_mac)); break; } } + + // Memorize that we have received a reply for this IP address + result[i].device[j].replied[scan_id]++; } return ret; } + // Convert netmask to CIDR static int netmask_to_cidr(struct in_addr *addr) { @@ -349,8 +353,9 @@ static void *arp_scan_iface(void *args) // Check if there are any results unsigned int replies = 0; for(unsigned int i = 0; i < arp_result_len; i++) - for(unsigned int j = 0; j < NUM_SCANS; j++) - replies += result[i].replied[j]; + for(unsigned int j = 0; j < MAX_MACS; j++) + for(unsigned int k = 0; k < NUM_SCANS; k++) + replies += result[i].device[j].replied[k]; if(pthread_mutex_lock(&lock) != 0) return NULL; @@ -366,51 +371,52 @@ static void *arp_scan_iface(void *args) printf("%-20s %-16s %-17s Reply matrix\n", "IP address", "Interface", "MAC address"); for(unsigned int i = 0; i < arp_result_len; i++) { - // Check if IP address replied - bool replied = false, multiple_replies = false; - for(unsigned int j = 0; j < NUM_SCANS; j++) + unsigned int j = 0, replied_devices = 0; + bool multiple_replies = false; + + // Print MAC addresses + for(j = 0; j < MAX_MACS; j++) { - if(result[i].replied[j] > 0) + // Check if IP address replied + bool replied = false; + for(unsigned int k = 0; k < NUM_SCANS; k++) { - replied = true; - multiple_replies |= result[i].replied[j] > 1; + replied |= result[i].device[j].replied[k] > 0; + multiple_replies |= result[i].device[j].replied[k] > 1; } - } - if(!replied) - continue; + if(!replied) + continue; - // Convert IP address to string - struct in_addr ip = { 0 }; - ip.s_addr = htonl(ntohl(dst_addr.s_addr) + i); - inet_ntop(AF_INET, &ip, ipstr, INET_ADDRSTRLEN); + // Check if IP address replied multiple times from different MAC address + replied_devices++; - // Print MAC addresses - unsigned int j = 0; - for(j = 0; j < MAX_MACS; j++) - { + // Convert IP address to string + struct in_addr ip = { 0 }; + ip.s_addr = htonl(ntohl(dst_addr.s_addr) + i); + inet_ntop(AF_INET, &ip, ipstr, INET_ADDRSTRLEN); // Check if result[i].mac[j] is all-zero - if(memcmp(result[i].mac[j], "\x00\x00\x00\x00\x00\x00", 6) == 0) + if(memcmp(result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", 6) == 0) break; // Print MAC address printf("%-20s %-16s %02x:%02x:%02x:%02x:%02x:%02x ", ipstr, iface, - result[i].mac[j][0], - result[i].mac[j][1], - result[i].mac[j][2], - result[i].mac[j][3], - result[i].mac[j][4], - result[i].mac[j][5]); + result[i].device[j].mac[0], + result[i].device[j].mac[1], + result[i].device[j].mac[2], + result[i].device[j].mac[3], + result[i].device[j].mac[4], + result[i].device[j].mac[5]); for(unsigned int k = 0; k < NUM_SCANS; k++) { - printf(" %s", result[i].replied[k] > 0 ? "X" : "-"); + printf(" %s", result[i].device[j].replied[k] > 0 ? "X" : "-"); } putc('\n', stdout); } // Print warning if we received multiple replies - if(j > 1 || multiple_replies) + if(replied_devices > 1 || multiple_replies) printf("WARNING: Received multiple replies for %s\n", ipstr); } putc('\n', stdout); From 65d5b121278cb57972f13c483f3a0a8f5b61b181 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 May 2023 07:34:18 +0200 Subject: [PATCH 35/78] Add our own address to the scan results so we can detect IP conflicts also here Signed-off-by: DL6ER --- src/tools/arp-scan.c | 75 ++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 09aace8c4..76430002f 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -181,6 +181,40 @@ static int create_arp_socket(const int ifindex, const char *iface) return arp_socket; } +static void add_result(const char *iface, struct in_addr *rcv_ip, struct in_addr *dst_ip, unsigned char *sender_mac, + struct arp_result *result, const size_t result_len, const unsigned int scan_id) +{ + + // Check if we have already found this IP address + uint32_t i = ntohl(rcv_ip->s_addr) - ntohl(dst_ip->s_addr); + if(i >= result_len) + { + printf("Received IP address %s out of range\n", inet_ntoa(*rcv_ip)); + return; + } + + // Save MAC address + unsigned int j = 0; + for(; j < MAX_MACS; j++) + { + // Check if received MAC is already stored in result[i].device[j].mac + if(memcmp(result[i].device[j].mac, sender_mac, MAC_LENGTH) == 0) + { + break; + } + // Check if result[i].device[j].mac is all-zero + if(memcmp(result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", MAC_LENGTH) == 0) + { + // Copy MAC address to result[i].device[j].mac + memcpy(result[i].device[j].mac, sender_mac, MAC_LENGTH); + break; + } + } + + // Memorize that we have received a reply for this IP address + result[i].device[j].replied[scan_id]++; +} + // Read all ARP responses static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, struct arp_result *result, const size_t result_len, const unsigned int scan_id) @@ -237,41 +271,12 @@ static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, arp_resp->sender_mac[4], arp_resp->sender_mac[5]); #endif - - // Check if we have already found this IP address - uint32_t i = ntohl(sender_a.s_addr) - ntohl(dst_ip->s_addr); - if(i >= result_len) - { - printf("Received IP address %s out of range\n", inet_ntoa(sender_a)); - continue; - } - - // Save MAC address - unsigned int j = 0; - for(; j < MAX_MACS; j++) - { - // Check if received MAC is already stored in result[i].device[j].mac - if(memcmp(result[i].device[j].mac, arp_resp->sender_mac, MAC_LENGTH) == 0) - { - break; - } - // Check if result[i].device[j].mac is all-zero - if(memcmp(result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", MAC_LENGTH) == 0) - { - // Copy MAC address to result[i].device[j].mac - memcpy(result[i].device[j].mac, arp_resp->sender_mac, sizeof(arp_resp->sender_mac)); - break; - } - } - - // Memorize that we have received a reply for this IP address - result[i].device[j].replied[scan_id]++; + add_result(iface, &sender_a, dst_ip, arp_resp->sender_mac, result, result_len, scan_id); } return ret; } - // Convert netmask to CIDR static int netmask_to_cidr(struct in_addr *addr) { @@ -360,15 +365,23 @@ static void *arp_scan_iface(void *args) if(pthread_mutex_lock(&lock) != 0) return NULL; + // Exit early if there are no results if(replies == 0) { printf("No devices found on interface %s (%s/%i)\n", iface, ipstr, cidr); goto arp_scan_iface_end; } - // Print results + // If there is at least one result, print header printf("ARP scan on interface %s (%s/%i) finished\n", iface, ipstr, cidr); printf("%-20s %-16s %-17s Reply matrix\n", "IP address", "Interface", "MAC address"); + + // Add our own IP address to the results so IP conflicts can be detected + // (our own IP address is not included in the ARP scan) + for(unsigned int i = 0; i < NUM_SCANS; i++) + add_result(iface, &src_addr.sin_addr, &dst_addr, mac, result, arp_result_len, i); + + // Print results for(unsigned int i = 0; i < arp_result_len; i++) { unsigned int j = 0, replied_devices = 0; From 0e74485c15dfb5d2b60203e2286ff308c0373a88 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 May 2023 07:49:42 +0200 Subject: [PATCH 36/78] Include hostnames (if available) Signed-off-by: DL6ER --- src/tools/arp-scan.c | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 76430002f..e549cfa2c 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -25,8 +25,14 @@ // This is also the limit for interfaces // we scan for DHCP activity. #define MAXTHREADS 32 + +// How many MAC addresses do we store per IP address? #define MAX_MACS 3 + +// How many ARP requests do we send per IP address? #define NUM_SCANS 10 + +// How long do we wait for ARP replies in each scan [seconds]? #define ARP_TIMEOUT 1 // Global lock used by all threads @@ -34,6 +40,7 @@ static pthread_mutex_t lock; static bool arp_verbose = false; static bool arp_all = false; +// Protocol definitions #define PROTO_ARP 0x0806 #define ETH2_HEADER_LEN 14 #define HW_TYPE 1 @@ -43,10 +50,9 @@ static bool arp_all = false; #define ARP_REPLY 0x02 #define BUF_SIZE 60 -#pragma pack(push, 1) - // ARP header struct // See https://en.wikipedia.org/wiki/Address_Resolution_Protocol#Packet_structure +#pragma pack(push, 1) struct arp_header { unsigned short hardware_type; unsigned short protocol_type; @@ -284,6 +290,21 @@ static int netmask_to_cidr(struct in_addr *addr) return __builtin_popcount(addr->s_addr); } +static const char *get_hostname(const struct in_addr *addr) +{ + // Get hostname + struct hostent *he = gethostbyaddr(&addr->s_addr, sizeof(addr->s_addr), AF_INET); + if(he == NULL) + return "N/A"; + + // Allow at most 24 characters for the hostname + static char hostname[25] = { 0 }; + strncpy(hostname, he->h_name, 24); + + // Return hostname + return hostname; +} + static void *arp_scan_iface(void *args) { // Get interface details @@ -374,7 +395,7 @@ static void *arp_scan_iface(void *args) // If there is at least one result, print header printf("ARP scan on interface %s (%s/%i) finished\n", iface, ipstr, cidr); - printf("%-20s %-16s %-17s Reply matrix\n", "IP address", "Interface", "MAC address"); + printf("%-16s %-10s %-24s %-17s Reply matrix\n", "IP address", "Interface", "Hostname", "MAC address"); // Add our own IP address to the results so IP conflicts can be detected // (our own IP address is not included in the ARP scan) @@ -412,8 +433,8 @@ static void *arp_scan_iface(void *args) break; // Print MAC address - printf("%-20s %-16s %02x:%02x:%02x:%02x:%02x:%02x ", - ipstr, iface, + printf("%-16s %-10s %-24s %02x:%02x:%02x:%02x:%02x:%02x ", + ipstr, iface, get_hostname(&ip), result[i].device[j].mac[0], result[i].device[j].mac[1], result[i].device[j].mac[2], @@ -422,9 +443,8 @@ static void *arp_scan_iface(void *args) result[i].device[j].mac[5]); for(unsigned int k = 0; k < NUM_SCANS; k++) - { printf(" %s", result[i].device[j].replied[k] > 0 ? "X" : "-"); - } + putc('\n', stdout); } From d6695ebd4259953470017aa4d5071f31231cae54 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 May 2023 09:10:34 +0200 Subject: [PATCH 37/78] Update embedded Lua to 5.4.6 Signed-off-by: DL6ER --- src/lua/lcorolib.c | 4 ++-- src/lua/linit.c | 3 --- src/lua/lstate.c | 10 +++++++++- src/lua/lua.c | 18 ++---------------- src/lua/lua.h | 7 ++++--- src/lua/luac.c | 4 +--- src/lua/lualib.h | 4 ---- 7 files changed, 18 insertions(+), 32 deletions(-) diff --git a/src/lua/lcorolib.c b/src/lua/lcorolib.c index 40b880b14..c64adf08a 100644 --- a/src/lua/lcorolib.c +++ b/src/lua/lcorolib.c @@ -76,7 +76,7 @@ static int luaB_auxwrap (lua_State *L) { if (l_unlikely(r < 0)) { /* error? */ int stat = lua_status(co); if (stat != LUA_OK && stat != LUA_YIELD) { /* error in the coroutine? */ - stat = lua_resetthread(co, L); /* close its tbc variables */ + stat = lua_closethread(co, L); /* close its tbc variables */ lua_assert(stat != LUA_OK); lua_xmove(co, L, 1); /* move error message to the caller */ } @@ -172,7 +172,7 @@ static int luaB_close (lua_State *L) { int status = auxstatus(L, co); switch (status) { case COS_DEAD: case COS_YIELD: { - status = lua_resetthread(co, L); + status = lua_closethread(co, L); if (status == LUA_OK) { lua_pushboolean(L, 1); return 1; diff --git a/src/lua/linit.c b/src/lua/linit.c index 9a5bcfdcc..69808f84f 100644 --- a/src/lua/linit.c +++ b/src/lua/linit.c @@ -50,9 +50,6 @@ static const luaL_Reg loadedlibs[] = { {LUA_MATHLIBNAME, luaopen_math}, {LUA_UTF8LIBNAME, luaopen_utf8}, {LUA_DBLIBNAME, luaopen_debug}, - /****** Pi-hole modification ******/ - {LUA_PIHOLELIBNAME, luaopen_pihole}, - /**********************************/ {NULL, NULL} }; diff --git a/src/lua/lstate.c b/src/lua/lstate.c index 1fbefb4b1..1e925e5ad 100644 --- a/src/lua/lstate.c +++ b/src/lua/lstate.c @@ -339,7 +339,7 @@ int luaE_resetthread (lua_State *L, int status) { } -LUA_API int lua_resetthread (lua_State *L, lua_State *from) { +LUA_API int lua_closethread (lua_State *L, lua_State *from) { int status; lua_lock(L); L->nCcalls = (from) ? getCcalls(from) : 0; @@ -349,6 +349,14 @@ LUA_API int lua_resetthread (lua_State *L, lua_State *from) { } +/* +** Deprecated! Use 'lua_closethread' instead. +*/ +LUA_API int lua_resetthread (lua_State *L) { + return lua_closethread(L, NULL); +} + + LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) { int i; lua_State *L; diff --git a/src/lua/lua.c b/src/lua/lua.c index f269c9973..0ff884545 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -20,10 +20,6 @@ #include "lauxlib.h" #include "lualib.h" -/** Pi-hole modification **/ -#include "ftl_lua.h" -/**************************/ - #if !defined(LUA_PROGNAME) #define LUA_PROGNAME "lua" @@ -215,9 +211,7 @@ static int dostring (lua_State *L, const char *s, const char *name) { /* ** Receives 'globname[=modname]' and runs 'globname = require(modname)'. */ -/************** Pi-hole modification ***************/ -int dolibrary (lua_State *L, char *globname) { -/***************************************************/ +static int dolibrary (lua_State *L, char *globname) { int status; char *modname = strchr(globname, '='); if (modname == NULL) /* no explicit name? */ @@ -645,12 +639,6 @@ static int pmain (lua_State *L) { if (handle_luainit(L) != LUA_OK) /* run LUA_INIT */ return 0; /* error running LUA_INIT */ } - - /************** Pi-hole modification ***************/ - // Load and enable libraries bundled with Pi-hole - ftl_lua_init(L); - /***************************************************/ - if (!runargs(L, argv, optlim)) /* execute arguments -e and -l */ return 0; /* something failed */ if (script > 0) { /* execute main script (if there is one) */ @@ -671,9 +659,7 @@ static int pmain (lua_State *L) { } -/******* Pi-hole modification ********/ -int lua_main (int argc, char **argv) { -/*************************************/ +int main (int argc, char **argv) { int status, result; lua_State *L = luaL_newstate(); /* create state */ if (L == NULL) { diff --git a/src/lua/lua.h b/src/lua/lua.h index 01927c6d9..fd16cf805 100644 --- a/src/lua/lua.h +++ b/src/lua/lua.h @@ -18,10 +18,10 @@ #define LUA_VERSION_MAJOR "5" #define LUA_VERSION_MINOR "4" -#define LUA_VERSION_RELEASE "5" +#define LUA_VERSION_RELEASE "6" #define LUA_VERSION_NUM 504 -#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 5) +#define LUA_VERSION_RELEASE_NUM (LUA_VERSION_NUM * 100 + 6) #define LUA_VERSION "Lua " LUA_VERSION_MAJOR "." LUA_VERSION_MINOR #define LUA_RELEASE LUA_VERSION "." LUA_VERSION_RELEASE @@ -163,7 +163,8 @@ extern const char lua_ident[]; LUA_API lua_State *(lua_newstate) (lua_Alloc f, void *ud); LUA_API void (lua_close) (lua_State *L); LUA_API lua_State *(lua_newthread) (lua_State *L); -LUA_API int (lua_resetthread) (lua_State *L, lua_State *from); +LUA_API int (lua_closethread) (lua_State *L, lua_State *from); +LUA_API int (lua_resetthread) (lua_State *L); /* Deprecated! */ LUA_API lua_CFunction (lua_atpanic) (lua_State *L, lua_CFunction panicf); diff --git a/src/lua/luac.c b/src/lua/luac.c index 139468cc6..5f4a141bd 100644 --- a/src/lua/luac.c +++ b/src/lua/luac.c @@ -193,9 +193,7 @@ static int pmain(lua_State* L) return 0; } -/******* Pi-hole modification ********/ -int luac_main(int argc, char* argv[]) -/*************************************/ +int main(int argc, char* argv[]) { lua_State* L; int i=doargs(argc,argv); diff --git a/src/lua/lualib.h b/src/lua/lualib.h index a7cab4eb0..262552907 100644 --- a/src/lua/lualib.h +++ b/src/lua/lualib.h @@ -44,10 +44,6 @@ LUAMOD_API int (luaopen_debug) (lua_State *L); #define LUA_LOADLIBNAME "package" LUAMOD_API int (luaopen_package) (lua_State *L); -/************ Pi-hole modification *************/ -#define LUA_PIHOLELIBNAME "pihole" -LUAMOD_API int (luaopen_pihole) (lua_State *L); -/***********************************************/ /* open all previous libraries */ LUALIB_API void (luaL_openlibs) (lua_State *L); From ec72408cc33a6ece90002120409fe3ceb6119688 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 May 2023 09:11:02 +0200 Subject: [PATCH 38/78] Re-apply Pi-hole specific Lua patches Signed-off-by: DL6ER --- src/lua/linit.c | 3 +++ src/lua/lua.c | 18 ++++++++++++++++-- src/lua/luac.c | 4 +++- src/lua/lualib.h | 4 ++++ 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/lua/linit.c b/src/lua/linit.c index 69808f84f..9a5bcfdcc 100644 --- a/src/lua/linit.c +++ b/src/lua/linit.c @@ -50,6 +50,9 @@ static const luaL_Reg loadedlibs[] = { {LUA_MATHLIBNAME, luaopen_math}, {LUA_UTF8LIBNAME, luaopen_utf8}, {LUA_DBLIBNAME, luaopen_debug}, + /****** Pi-hole modification ******/ + {LUA_PIHOLELIBNAME, luaopen_pihole}, + /**********************************/ {NULL, NULL} }; diff --git a/src/lua/lua.c b/src/lua/lua.c index 0ff884545..f269c9973 100644 --- a/src/lua/lua.c +++ b/src/lua/lua.c @@ -20,6 +20,10 @@ #include "lauxlib.h" #include "lualib.h" +/** Pi-hole modification **/ +#include "ftl_lua.h" +/**************************/ + #if !defined(LUA_PROGNAME) #define LUA_PROGNAME "lua" @@ -211,7 +215,9 @@ static int dostring (lua_State *L, const char *s, const char *name) { /* ** Receives 'globname[=modname]' and runs 'globname = require(modname)'. */ -static int dolibrary (lua_State *L, char *globname) { +/************** Pi-hole modification ***************/ +int dolibrary (lua_State *L, char *globname) { +/***************************************************/ int status; char *modname = strchr(globname, '='); if (modname == NULL) /* no explicit name? */ @@ -639,6 +645,12 @@ static int pmain (lua_State *L) { if (handle_luainit(L) != LUA_OK) /* run LUA_INIT */ return 0; /* error running LUA_INIT */ } + + /************** Pi-hole modification ***************/ + // Load and enable libraries bundled with Pi-hole + ftl_lua_init(L); + /***************************************************/ + if (!runargs(L, argv, optlim)) /* execute arguments -e and -l */ return 0; /* something failed */ if (script > 0) { /* execute main script (if there is one) */ @@ -659,7 +671,9 @@ static int pmain (lua_State *L) { } -int main (int argc, char **argv) { +/******* Pi-hole modification ********/ +int lua_main (int argc, char **argv) { +/*************************************/ int status, result; lua_State *L = luaL_newstate(); /* create state */ if (L == NULL) { diff --git a/src/lua/luac.c b/src/lua/luac.c index 5f4a141bd..139468cc6 100644 --- a/src/lua/luac.c +++ b/src/lua/luac.c @@ -193,7 +193,9 @@ static int pmain(lua_State* L) return 0; } -int main(int argc, char* argv[]) +/******* Pi-hole modification ********/ +int luac_main(int argc, char* argv[]) +/*************************************/ { lua_State* L; int i=doargs(argc,argv); diff --git a/src/lua/lualib.h b/src/lua/lualib.h index 262552907..a7cab4eb0 100644 --- a/src/lua/lualib.h +++ b/src/lua/lualib.h @@ -44,6 +44,10 @@ LUAMOD_API int (luaopen_debug) (lua_State *L); #define LUA_LOADLIBNAME "package" LUAMOD_API int (luaopen_package) (lua_State *L); +/************ Pi-hole modification *************/ +#define LUA_PIHOLELIBNAME "pihole" +LUAMOD_API int (luaopen_pihole) (lua_State *L); +/***********************************************/ /* open all previous libraries */ LUALIB_API void (luaL_openlibs) (lua_State *L); From 2931fa381749170ca76694ac87076a9e2b1a758d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 May 2023 09:44:57 +0200 Subject: [PATCH 39/78] Skip ABP extended CSS selectors (port of core PR #5247) Signed-off-by: DL6ER --- src/gravity-tools.c | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index f93e74641..8047e917c 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -29,6 +29,9 @@ // See https://github.com/pi-hole/pi-hole/pull/5240 #define ABP_DOMAIN_REXEX "\\|\\|"SUBDOMAIN_PATTERN"*"TLD_PATTERN"\\^" +// Detects ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter +#define ABP_CSS_SELECTORS "[a-z]#[$?@]{0,1}#" + // A list of items of common local hostnames not to report as unusable // Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files // but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output @@ -71,7 +74,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis rewind(fpin); // Compile regular expression to validate domains - regex_t exact_regex, abp_regex, false_positives_regex; + regex_t exact_regex, abp_regex, false_positives_regex, abp_css_regex; if(regcomp(&exact_regex, VALID_DOMAIN_REXEX, REG_EXTENDED) != 0) { printf("\r %s Unable to compile regular expression to validate exact domains\n", cross); @@ -93,6 +96,13 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis sqlite3_close(db); return EXIT_FAILURE; } + if(regcomp(&abp_css_regex, ABP_CSS_SELECTORS, REG_EXTENDED) != 0) + { + printf("\r %s Unable to compile regular expression to validate ABP-style domains\n", cross); + fclose(fpin); + sqlite3_close(db); + return EXIT_FAILURE; + } // Begin transaction if(sqlite3_exec(db, "BEGIN TRANSACTION;", NULL, NULL, NULL) != SQLITE_OK) @@ -139,14 +149,27 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis total_read += read; lineno++; + // Remove leading whitespace + // isspace() matches spaces, tabs, form feeds, line breaks, + // carriage returns, and vertical tabs + while(isspace(line[0])) + line++; + // Skip comments - if(line[0] == '#') + // # is used for comments in HOSTS files + // ! is used for comments in AdBlock-style files + // [ is used for comments in AdGuard-style files and ABP headers + if(line[0] == '#' || line[0] == '!' || line[0] == '[') continue; // Remove trailing newline if(line[read-1] == '\n') line[--read] = '\0'; + // Remove trailing carriage return (Windows) + if(line[read-1] == '\r') + line[--read] = '\0'; + // Remove trailing dot (convert FQDN to domain) if(line[read-1] == '.') line[--read] = '\0'; @@ -186,6 +209,14 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis match.rm_so == 0 && match.rm_eo == read) // <- Match covers entire line { // ABP-style match (see comments above) + + // Skip lines containing ABP extended CSS selectors + // ("##", "#!#", "#@#", "#?#") preceded by a letter + // See https://github.com/pi-hole/pi-hole/pull/5247 for + // further information on why this is necessary + if(regexec(&abp_css_regex, line, 1, &match, 0) == 0) + continue; + // Append pattern to database using prepared statement if(sqlite3_bind_text(stmt, 1, line, -1, SQLITE_STATIC) != SQLITE_OK) { @@ -336,6 +367,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis free(line); regfree(&exact_regex); regfree(&abp_regex); + regfree(&false_positives_regex); + regfree(&abp_css_regex); for(unsigned int i = 0; i < invalid_domains_list_len; i++) if(invalid_domains_list[i] != NULL) free(invalid_domains_list[i]); From ff1437eb4f1e1b3aa6a648253c7a0b2141675f84 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 14 May 2023 09:56:41 +0200 Subject: [PATCH 40/78] Spellcheck correction Signed-off-by: DL6ER --- src/gravity-tools.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index 8047e917c..d3091f186 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -16,7 +16,7 @@ // Define valid domain patterns // No need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already // Adapted from https://stackoverflow.com/a/30007882 -// - Added "(?:...)" to form non-capturing groups (slighly faster) +// - Added "(?:...)" to form non-capturing groups (slightly faster) #define TLD_PATTERN "[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" #define SUBDOMAIN_PATTERN "([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\\.)" From 6913775e8edc80307d4b5ca9f30b90a565046447 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 May 2023 19:15:59 +0200 Subject: [PATCH 41/78] Use OVER constant instead of carridge return Signed-off-by: DL6ER --- src/args.c | 10 ++++++- src/args.h | 1 + src/gravity-tools.c | 68 +++++++++++++++++++++++++++++---------------- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/src/args.c b/src/args.c index 8c7b811d4..ffe2a726e 100644 --- a/src/args.c +++ b/src/args.c @@ -126,7 +126,15 @@ static const char __attribute__ ((pure)) *cli_color(const char *color) return is_term() ? color : ""; } -static inline bool strEndsWith(const char *input, const char *end){ +// Go back to beginning of line and erase to end of line if STDOUT is a terminal +const char __attribute__ ((pure)) *cli_over(void) +{ + // \x1b[K is the ANSI escape sequence for "erase to end of line" + return is_term() ? "\r\x1b[K" : "\r"; +} + +static inline bool strEndsWith(const char *input, const char *end) +{ return strcmp(input + strlen(input) - strlen(end), end) == 0; } diff --git a/src/args.h b/src/args.h index 57371e4e6..e92b07a6c 100644 --- a/src/args.h +++ b/src/args.h @@ -23,6 +23,7 @@ const char *cli_qst(void) __attribute__ ((const)); const char *cli_done(void) __attribute__ ((pure)); const char *cli_bold(void) __attribute__ ((pure)); const char *cli_normal(void) __attribute__ ((pure)); +const char *cli_over(void) __attribute__ ((pure)); // defined in dnsmasq_interface.c int check_struct_sizes(void); diff --git a/src/gravity-tools.c b/src/gravity-tools.c index d3091f186..523fec54a 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -50,12 +50,13 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis const char *info = cli_info(); const char *tick = cli_tick(); const char *cross = cli_cross(); + const char *over = cli_over(); // Open input file FILE *fpin = fopen(infile, "r"); if(fpin == NULL) { - printf("\r %s Unable to open %s for reading\n", cross, infile); + printf("%s %s Unable to open %s for reading\n", over, cross, infile); return EXIT_FAILURE; } @@ -64,7 +65,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis sqlite3_stmt *stmt = NULL; if(sqlite3_open_v2(outfile, &db, SQLITE_OPEN_READWRITE, NULL) != SQLITE_OK) { - printf("\r %s Unable to open database file %s for writing\n", cross, outfile); + printf("%s %s Unable to open database file %s for writing\n", over, cross, outfile); fclose(fpin); return EXIT_FAILURE; } @@ -77,28 +78,32 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis regex_t exact_regex, abp_regex, false_positives_regex, abp_css_regex; if(regcomp(&exact_regex, VALID_DOMAIN_REXEX, REG_EXTENDED) != 0) { - printf("\r %s Unable to compile regular expression to validate exact domains\n", cross); + printf("%s %s Unable to compile regular expression to validate exact domains\n", + over, cross); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } if(regcomp(&abp_regex, ABP_DOMAIN_REXEX, REG_EXTENDED) != 0) { - printf("\r %s Unable to compile regular expression to validate ABP-style domains\n", cross); + printf("%s %s Unable to compile regular expression to validate ABP-style domains\n", + over, cross); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } if(regcomp(&false_positives_regex, FALSE_POSITIVES, REG_EXTENDED | REG_NOSUB) != 0) { - printf("\r %s Unable to compile regular expression to identify false positives\n", cross); + printf("%s %s Unable to compile regular expression to identify false positives\n", + over, cross); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } if(regcomp(&abp_css_regex, ABP_CSS_SELECTORS, REG_EXTENDED) != 0) { - printf("\r %s Unable to compile regular expression to validate ABP-style domains\n", cross); + printf("%s %s Unable to compile regular expression to validate ABP-style domains\n", + over, cross); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -107,7 +112,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Begin transaction if(sqlite3_exec(db, "BEGIN TRANSACTION;", NULL, NULL, NULL) != SQLITE_OK) { - printf("\r %s Unable to begin transaction to insert domains into database file %s\n", cross, outfile); + printf("%s %s Unable to begin transaction to insert domains into database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -117,7 +123,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis const char *sql = "INSERT INTO gravity (domain, adlist_id) VALUES (?, ?);"; if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { - printf("\r %s Unable to prepare SQL statement to insert domains into database file %s\n", cross, outfile); + printf("%s %s Unable to prepare SQL statement to insert domains into database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -127,7 +134,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis const int adlistID = atoi(adlistIDstr); if(sqlite3_bind_int(stmt, 2, adlistID) != SQLITE_OK) { - printf("\r %s Unable to bind adlistID to SQL statement to insert domains into database file %s\n", cross, outfile); + printf("%s %s Unable to bind adlistID to SQL statement to insert domains into database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -188,14 +196,15 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Append domain to database using prepared statement if(sqlite3_bind_text(stmt, 1, line, -1, SQLITE_STATIC) != SQLITE_OK) { - printf("\r %s Unable to bind domain to SQL statement to insert domains into database file %s\n", cross, outfile); + printf("%s %s Unable to bind domain to SQL statement to insert domains into database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } if(sqlite3_step(stmt) != SQLITE_DONE) { - printf("\r %s Unable to insert domain into database file %s\n", cross, outfile); + printf("%s %s Unable to insert domain into database file %s\n", over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -220,14 +229,15 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Append pattern to database using prepared statement if(sqlite3_bind_text(stmt, 1, line, -1, SQLITE_STATIC) != SQLITE_OK) { - printf("\r %s Unable to bind domain to SQL statement to insert domains into database file %s\n", cross, outfile); + printf("%s %s Unable to bind domain to SQL statement to insert domains into database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } if(sqlite3_step(stmt) != SQLITE_DONE) { - printf("\r %s Unable to insert domain into database file %s\n", cross, outfile); + printf("%s %s Unable to insert domain into database file %s\n", over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -267,7 +277,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Print progress if it has changed if(progress > last_progress) { - printf("\r %s Processed %i%% of downloaded list", info, progress); + printf("%s %s Processed %i%% of downloaded list", over, info, progress); fflush(stdout); last_progress = progress; } @@ -277,7 +287,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Finalize SQL statement if(sqlite3_finalize(stmt) != SQLITE_OK) { - printf("\r %s Unable to finalize SQL statement to insert domains into database file %s\n", cross, outfile); + printf("%s %s Unable to finalize SQL statement to insert domains into database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -290,7 +301,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis sql = "INSERT OR REPLACE INTO info (property,value) VALUES ('abp_domains',1);"; if(sqlite3_exec(db, sql, NULL, NULL, NULL) != SQLITE_OK) { - printf("\r %s Unable to update database properties in database file %s\n", cross, outfile); + printf("%s %s Unable to update database properties in database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -301,7 +313,8 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis sql = "UPDATE adlist SET number = ?, invalid_domains = ? WHERE id = ?;"; if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { - printf("\r %s Unable to prepare SQL statement to update adlist properties in database file %s\n", cross, outfile); + printf("%s %s Unable to prepare SQL statement to update adlist properties in database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -310,35 +323,40 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Update date if(sqlite3_bind_int(stmt, 1, exact_domains) != SQLITE_OK) { - printf("\r %s Unable to bind number of domains to SQL statement to update adlist properties in database file %s\n", cross, outfile); + printf("%s %s Unable to bind number of domains to SQL statement to update adlist properties in database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } if(sqlite3_bind_int(stmt, 2, invalid_domains) != SQLITE_OK) { - printf("\r %s Unable to bind number of invalid domains to SQL statement to update adlist properties in database file %s\n", cross, outfile); + printf("%s %s Unable to bind number of invalid domains to SQL statement to update adlist properties in database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } if(sqlite3_bind_int(stmt, 3, adlistID) != SQLITE_OK) { - printf("\r %s Unable to bind adlist ID to SQL statement to update adlist properties in database file %s\n", cross, outfile); + printf("%s %s Unable to bind adlist ID to SQL statement to update adlist properties in database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } if(sqlite3_step(stmt) != SQLITE_DONE) { - printf("\r %s Unable to update adlist properties in database file %s\n", cross, outfile); + printf("%s %s Unable to update adlist properties in database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } if(sqlite3_finalize(stmt) != SQLITE_OK) { - printf("\r %s Unable to finalize SQL statement to update adlist properties in database file %s\n", cross, outfile); + printf("%s %s Unable to finalize SQL statement to update adlist properties in database file %s\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; @@ -347,14 +365,16 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // End transaction if(sqlite3_exec(db, "END TRANSACTION", NULL, NULL, NULL) != SQLITE_OK) { - printf("\r %s Unable to end transaction to insert domains into database file %s (database file may be corrupted)\n", cross, outfile); + printf("%s %s Unable to end transaction to insert domains into database file %s (database file may be corrupted)\n", + over, cross, outfile); fclose(fpin); sqlite3_close(db); return EXIT_FAILURE; } // Print summary - printf("\r %s Parsed %u exact domains and %u ABP-style domains (ignored %u non-domain entries)\n", tick, exact_domains, abp_domains, invalid_domains); + printf("%s %s Parsed %u exact domains and %u ABP-style domains (ignored %u non-domain entries)\n", + over, tick, exact_domains, abp_domains, invalid_domains); if(invalid_domains_list_len > 0) { puts(" Sample of non-domain entries:"); From 041092a6f4d033aa874a7520444f2adf952bd5a9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 May 2023 19:50:46 +0200 Subject: [PATCH 42/78] Print progress in verbose arp-scanning mode Signed-off-by: DL6ER --- src/tools/arp-scan.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index e549cfa2c..3d7ccb8dc 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -364,9 +364,9 @@ static void *arp_scan_iface(void *args) for(unsigned int scan_id = 0; scan_id < NUM_SCANS; scan_id++) { -#ifdef DEBUG - printf("Scanning interface %s (%s/%i) for the %i. time\n", iface, ipstr, cidr, scan_id + 1); -#endif + if(arp_verbose) + printf("Still scanning interface %s (%s/%i) %i%%...\n", iface, ipstr, cidr, 100*scan_id/NUM_SCANS); + // Send ARP requests to all IPs in subnet if(send_arps(arp_socket, ifindex, iface, mac, &src_addr.sin_addr, dst_addr, cidr) != 0) break; From 1e0f20b2bc7306561d06dcca0ea89746787deafc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 May 2023 19:54:52 +0200 Subject: [PATCH 43/78] Print different warnings if we received multiple replies from (apparently) the same device or if we received replies for the same address from different MAC addresses Signed-off-by: DL6ER --- src/tools/arp-scan.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 3d7ccb8dc..ffbf73788 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -406,7 +406,7 @@ static void *arp_scan_iface(void *args) for(unsigned int i = 0; i < arp_result_len; i++) { unsigned int j = 0, replied_devices = 0; - bool multiple_replies = false; + unsigned int multiple_replies = 0; // Print MAC addresses for(j = 0; j < MAX_MACS; j++) @@ -416,7 +416,7 @@ static void *arp_scan_iface(void *args) for(unsigned int k = 0; k < NUM_SCANS; k++) { replied |= result[i].device[j].replied[k] > 0; - multiple_replies |= result[i].device[j].replied[k] > 1; + multiple_replies += result[i].device[j].replied[k] > 1; } if(!replied) continue; @@ -449,8 +449,12 @@ static void *arp_scan_iface(void *args) } // Print warning if we received multiple replies - if(replied_devices > 1 || multiple_replies) - printf("WARNING: Received multiple replies for %s\n", ipstr); + if(replied_devices > 1) + printf("WARNING: Received replies for %s from %i devices\n", + ipstr, replied_devices); + if(multiple_replies > 0) + printf("WARNING: Received multiple replies for %s in %i scan%s\n", + ipstr, multiple_replies, multiple_replies > 1 ? "s" : ""); } putc('\n', stdout); From f1b4a338f04aa48536e91587b2e7e86b9885927a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 May 2023 20:58:37 +0200 Subject: [PATCH 44/78] Consolidate output in main process Signed-off-by: DL6ER --- src/args.c | 6 +- src/tools/arp-scan.c | 248 +++++++++++++++++++++++++++++-------------- src/tools/arp-scan.h | 2 +- 3 files changed, 172 insertions(+), 84 deletions(-) diff --git a/src/args.c b/src/args.c index f86a19e30..14b4031d5 100644 --- a/src/args.c +++ b/src/args.c @@ -178,9 +178,8 @@ void parse_args(int argc, char* argv[]) { // Enable stdout printing cli_mode = true; - const bool verbose = argc > 2 && strcmp(argv[2], "-v") == 0; const bool arp_all = argc > 2 && strcmp(argv[2], "-a") == 0; - exit(run_arp_scan(verbose, arp_all)); + exit(run_arp_scan(arp_all)); } // start from 1, as argv[0] is the executable name @@ -514,9 +513,8 @@ void parse_args(int argc, char* argv[]) printf("%sOther:%s\n", yellow, normal); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); - printf("\t%sarp-scan %s[{-v/-a}]%s Use ARP to scan local network for\n", green, cyan, normal); + printf("\t%sarp-scan %s[-a]%s Use ARP to scan local network for\n", green, cyan, normal); printf("\t possible IP conflicts\n"); - printf("\t Append %s-v%s for verbose output mode\n", cyan, normal); printf("\t Append %s-a%s to force scan on all\n", cyan, normal); printf("\t interfaces\n"); printf("\t%s-h%s, %shelp%s Display this help and exit\n\n", green, normal, green, normal); diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index ffbf73788..69e702412 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -14,6 +14,8 @@ #include "log.h" // get_hardware_address() #include "dhcp-discover.h" +// sleepms() +#include "timers.h" #include #include @@ -35,9 +37,7 @@ // How long do we wait for ARP replies in each scan [seconds]? #define ARP_TIMEOUT 1 -// Global lock used by all threads -static pthread_mutex_t lock; -static bool arp_verbose = false; +// Global constant static bool arp_all = false; // Protocol definitions @@ -73,6 +73,30 @@ struct arp_result { } device[MAX_MACS]; }; +enum status { + STATUS_INITIALIZING = 0, + STATUS_SKIPPED_CIDR_MISMATCH, + STATUS_SCANNING, + STATUS_ERROR, + STATUS_COMPLETE +}; + +struct thread_data { + int dst_cidr; + struct sockaddr_in src_addr; + struct sockaddr_in dst_addr; + struct sockaddr_in mask; + struct ifaddrs *ifa; + const char *iface; + struct arp_result *result; + size_t result_size; + enum status status; + char ipstr[INET_ADDRSTRLEN]; + unsigned char mac[16]; + unsigned int num_scans; + char *error; +}; + // Sends multiple ARP who-has request on interface ifindex, using source mac src_mac and source ip src_ip. // Iterates over all IP addresses in the range of dst_ip/cidr. static int send_arps(const int fd, const int ifindex, const char *iface, const unsigned char *src_mac, @@ -195,7 +219,7 @@ static void add_result(const char *iface, struct in_addr *rcv_ip, struct in_addr uint32_t i = ntohl(rcv_ip->s_addr) - ntohl(dst_ip->s_addr); if(i >= result_len) { - printf("Received IP address %s out of range\n", inet_ntoa(*rcv_ip)); + printf("Received IP address %s out of range for interface %s (%u >= %zu)\n", inet_ntoa(*rcv_ip), iface, i, result_len); return; } @@ -307,103 +331,132 @@ static const char *get_hostname(const struct in_addr *addr) static void *arp_scan_iface(void *args) { + // Get thread_data pointer + struct thread_data *thread_data = (struct thread_data*)args; + // Get interface details - struct ifaddrs *ifa = (struct ifaddrs*)args; + struct ifaddrs *ifa = thread_data->ifa; // Get interface name - const char *iface = ifa->ifa_name; + const char *iface = thread_data->iface; // Set interface name as thread name prctl(PR_SET_NAME, iface, 0, 0, 0); // Get interface IPv4 address - struct sockaddr_in src_addr = { 0 }; - memcpy(&src_addr, ((struct ifaddrs*)args)->ifa_addr, sizeof(src_addr)); - char ipstr[INET_ADDRSTRLEN] = { 0 }; - inet_ntop(AF_INET, &src_addr.sin_addr, ipstr, INET_ADDRSTRLEN); + memcpy(&thread_data->src_addr, ifa->ifa_addr, sizeof(thread_data->src_addr)); + inet_ntop(AF_INET, &thread_data->src_addr.sin_addr, thread_data->ipstr, INET_ADDRSTRLEN); // Get interface netmask - struct sockaddr_in mask = { 0 }; - memcpy(&mask, ((struct ifaddrs*)args)->ifa_netmask, sizeof(mask)); - // char netmask[INET_ADDRSTRLEN] = { 0 }; - // inet_ntop(AF_INET, &mask.sin_addr, netmask, INET_ADDRSTRLEN); + memcpy(&thread_data->mask, ifa->ifa_netmask, sizeof(thread_data->mask)); // Convert subnet to CIDR - const int cidr = netmask_to_cidr(&mask.sin_addr); + thread_data->dst_cidr = netmask_to_cidr(&thread_data->mask.sin_addr); // Get interface index const int ifindex = if_nametoindex(iface); // Scan only interfaces with CIDR >= 24 - if(cidr < 24 && !arp_all) + if(thread_data->dst_cidr < 24 && !arp_all) { - printf("Skipped interface %s (%s/%i)\n", iface, ipstr, cidr); + thread_data->status = STATUS_SKIPPED_CIDR_MISMATCH; + //printf("Skipped interface %s (%s/%i)\n", iface, thread_data->ipstr, thread_data->dst_cidr); pthread_exit(NULL); } - if(arp_verbose) - printf("Scanning interface %s (%s/%i)...\n", iface, ipstr, cidr); + //if(arp_verbose) + // printf("Scanning interface %s (%s/%i)...\n", iface, thread_data->ipstr, thread_data->dst_cidr); + thread_data->status = STATUS_SCANNING; // Create socket for ARP communications const int arp_socket = create_arp_socket(ifindex, iface); // Cannot create socket, likely a permission error if(arp_socket < 0) + { + thread_data->status = STATUS_ERROR; pthread_exit(NULL); + } // Get hardware address of client machine - unsigned char mac[16] = { 0 }; - get_hardware_address(arp_socket, iface, mac); + get_hardware_address(arp_socket, iface, thread_data->mac); // Define destination IP address by masking source IP with netmask - struct in_addr dst_addr = { 0 }; - dst_addr.s_addr = src_addr.sin_addr.s_addr & mask.sin_addr.s_addr; + thread_data->dst_addr.sin_addr.s_addr = thread_data->src_addr.sin_addr.s_addr & thread_data->mask.sin_addr.s_addr; // Allocate memory for ARP response buffer - const size_t arp_result_len = 1 << (32 - cidr); - struct arp_result *result = calloc(arp_result_len, sizeof(struct arp_result)); + const size_t arp_result_len = 1 << (32 - thread_data->dst_cidr); + thread_data->result_size = arp_result_len; + struct arp_result *result = calloc(thread_data->result_size, sizeof(struct arp_result)); + thread_data->result = result; - for(unsigned int scan_id = 0; scan_id < NUM_SCANS; scan_id++) + for(thread_data->num_scans = 0; thread_data->num_scans < NUM_SCANS; thread_data->num_scans++) { - if(arp_verbose) - printf("Still scanning interface %s (%s/%i) %i%%...\n", iface, ipstr, cidr, 100*scan_id/NUM_SCANS); + //if(arp_verbose) + // printf("Still scanning interface %s (%s/%i) %i%%...\n", iface, thread_data->ipstr, thread_data->dst_cidr, 100*scan_id/NUM_SCANS); // Send ARP requests to all IPs in subnet - if(send_arps(arp_socket, ifindex, iface, mac, &src_addr.sin_addr, dst_addr, cidr) != 0) + if(send_arps(arp_socket, ifindex, iface, thread_data->mac, &thread_data->src_addr.sin_addr, thread_data->dst_addr.sin_addr, thread_data->dst_cidr) != 0) + { + thread_data->status = STATUS_ERROR; break; + } // Read ARP responses - if(read_arp(arp_socket, iface, &dst_addr, result, arp_result_len, scan_id) != 0) + if(read_arp(arp_socket, iface, &thread_data->dst_addr.sin_addr, thread_data->result, thread_data->result_size, thread_data->num_scans) != 0) + { + thread_data->status = STATUS_ERROR; break; + } + } + + // Close socket + if(close(arp_socket) != 0) + thread_data->status = STATUS_ERROR; + + if(thread_data->status != STATUS_ERROR) + thread_data->status = STATUS_COMPLETE; + + pthread_exit(NULL); +} + +static void print_results(struct thread_data *thread_data) +{ + + if(thread_data->status == STATUS_SKIPPED_CIDR_MISMATCH) + { + printf("Skipped interface %s (%s/%i) because of too large network (use -a to force scanning this interface)\n\n", + thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); + return; } // Check if there are any results unsigned int replies = 0; - for(unsigned int i = 0; i < arp_result_len; i++) + for(unsigned int i = 0; i < thread_data->result_size; i++) for(unsigned int j = 0; j < MAX_MACS; j++) for(unsigned int k = 0; k < NUM_SCANS; k++) - replies += result[i].device[j].replied[k]; - - if(pthread_mutex_lock(&lock) != 0) - return NULL; + replies += thread_data->result[i].device[j].replied[k]; // Exit early if there are no results if(replies == 0) { - printf("No devices found on interface %s (%s/%i)\n", iface, ipstr, cidr); - goto arp_scan_iface_end; + printf("No devices found on interface %s (%s/%i)\n\n", + thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); + return; } // If there is at least one result, print header - printf("ARP scan on interface %s (%s/%i) finished\n", iface, ipstr, cidr); - printf("%-16s %-10s %-24s %-17s Reply matrix\n", "IP address", "Interface", "Hostname", "MAC address"); + printf("ARP scan on interface %s (%s/%i) finished\n", + thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); + printf("%-16s %-10s %-24s %-17s Reply matrix\n", + "IP address", "Interface", "Hostname", "MAC address"); // Add our own IP address to the results so IP conflicts can be detected // (our own IP address is not included in the ARP scan) for(unsigned int i = 0; i < NUM_SCANS; i++) - add_result(iface, &src_addr.sin_addr, &dst_addr, mac, result, arp_result_len, i); + add_result(thread_data->iface, &thread_data->src_addr.sin_addr, &thread_data->dst_addr.sin_addr, thread_data->mac, thread_data->result, thread_data->result_size, i); // Print results - for(unsigned int i = 0; i < arp_result_len; i++) + for(unsigned int i = 0; i < thread_data->result_size; i++) { unsigned int j = 0, replied_devices = 0; unsigned int multiple_replies = 0; @@ -415,8 +468,8 @@ static void *arp_scan_iface(void *args) bool replied = false; for(unsigned int k = 0; k < NUM_SCANS; k++) { - replied |= result[i].device[j].replied[k] > 0; - multiple_replies += result[i].device[j].replied[k] > 1; + replied |= thread_data->result[i].device[j].replied[k] > 0; + multiple_replies += thread_data->result[i].device[j].replied[k] > 1; } if(!replied) continue; @@ -426,24 +479,25 @@ static void *arp_scan_iface(void *args) // Convert IP address to string struct in_addr ip = { 0 }; - ip.s_addr = htonl(ntohl(dst_addr.s_addr) + i); - inet_ntop(AF_INET, &ip, ipstr, INET_ADDRSTRLEN); + ip.s_addr = htonl(ntohl(thread_data->dst_addr.sin_addr.s_addr) + i); + inet_ntop(AF_INET, &ip, thread_data->ipstr, INET_ADDRSTRLEN); // Check if result[i].mac[j] is all-zero - if(memcmp(result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", 6) == 0) + if(memcmp(thread_data->result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", 6) == 0) break; // Print MAC address printf("%-16s %-10s %-24s %02x:%02x:%02x:%02x:%02x:%02x ", - ipstr, iface, get_hostname(&ip), - result[i].device[j].mac[0], - result[i].device[j].mac[1], - result[i].device[j].mac[2], - result[i].device[j].mac[3], - result[i].device[j].mac[4], - result[i].device[j].mac[5]); + thread_data->ipstr, thread_data->iface, + get_hostname(&ip), + thread_data->result[i].device[j].mac[0], + thread_data->result[i].device[j].mac[1], + thread_data->result[i].device[j].mac[2], + thread_data->result[i].device[j].mac[3], + thread_data->result[i].device[j].mac[4], + thread_data->result[i].device[j].mac[5]); for(unsigned int k = 0; k < NUM_SCANS; k++) - printf(" %s", result[i].device[j].replied[k] > 0 ? "X" : "-"); + printf(" %s", thread_data->result[i].device[j].replied[k] > 0 ? "X" : "-"); putc('\n', stdout); } @@ -451,25 +505,16 @@ static void *arp_scan_iface(void *args) // Print warning if we received multiple replies if(replied_devices > 1) printf("WARNING: Received replies for %s from %i devices\n", - ipstr, replied_devices); + thread_data->ipstr, replied_devices); if(multiple_replies > 0) printf("WARNING: Received multiple replies for %s in %i scan%s\n", - ipstr, multiple_replies, multiple_replies > 1 ? "s" : ""); + thread_data->ipstr, multiple_replies, multiple_replies > 1 ? "s" : ""); } putc('\n', stdout); - -arp_scan_iface_end: - if(pthread_mutex_unlock(&lock) != 0) - return NULL; - - // Close socket - close(arp_socket); - pthread_exit(NULL); } -int run_arp_scan(const bool verbose, const bool scan_all) +int run_arp_scan(const bool scan_all) { - arp_verbose = verbose; arp_all = scan_all; puts("Discovering IPv4 hosts on the network using the Address Resolution Protocol (ARP)...\n"); @@ -480,28 +525,25 @@ int run_arp_scan(const bool verbose, const bool scan_all) // Initialize thread attributes object with default attribute values pthread_attr_init(&attr); - // Create processing/logging lock - pthread_mutexattr_t lock_attr = {}; - // Initialize the lock attributes - pthread_mutexattr_init(&lock_attr); - // Initialize the lock - pthread_mutex_init(&lock, &lock_attr); - // Destroy the lock attributes since we're done with it - pthread_mutexattr_destroy(&lock_attr); - struct ifaddrs *addrs, *tmp; getifaddrs(&addrs); tmp = addrs; // Loop until there are no more interfaces available // or we reached the maximum number of threads - int tid = 0; + unsigned int tid = 0; + + struct thread_data thread_data[MAXTHREADS] = {0}; + while(tmp != NULL && tid < MAXTHREADS) { // Create a thread for interfaces of type AF_INET if(tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_INET) { - if(pthread_create(&scanthread[tid], &attr, arp_scan_iface, tmp ) != 0) + thread_data[tid].ifa = tmp; + thread_data[tid].iface = tmp->ifa_name; + // Create thread + if(pthread_create(&scanthread[tid], &attr, arp_scan_iface, &thread_data[tid] ) != 0) { printf("Unable to launch thread for interface %s, skipping...\n", tmp->ifa_name); @@ -516,12 +558,60 @@ int run_arp_scan(const bool verbose, const bool scan_all) tmp = tmp->ifa_next; } + // Wait for all threads to finish scanning + bool all_done = false; + while(!all_done) + { + all_done = true; + unsigned int num_scans = 0, total_scans = 0; + for(unsigned int i = 0; i < tid; i++) + { + if(thread_data[i].status == STATUS_SCANNING) + { + // At least one thread is still scanning + all_done = false; + num_scans += thread_data[i].num_scans; + total_scans += NUM_SCANS; + } + if(thread_data[i].status == STATUS_COMPLETE) + { + // Also add up scans for completed threads + num_scans += thread_data[i].num_scans; + total_scans += NUM_SCANS; + } + } + if(!all_done) + { + // Print progress + printf("%i%%... ", 100*num_scans/total_scans); + // Flush stdout + fflush(stdout); + // Sleep for 1 second + sleepms(1000); + } + } + puts("100%%\n\n"); + // Wait for all threads to join back with us - for(tid--; tid > -1; tid--) - pthread_join(scanthread[tid], NULL); + for(unsigned int i = 0; i < tid; i++) + pthread_join(scanthread[i], NULL); + + // Destroy the thread attributes object, since we are done with it + pthread_attr_destroy(&attr); // Free linked-list of interfaces on this client freeifaddrs(addrs); + // Loop over thread results and print them + for(unsigned int i = 0; i < tid; i++) + { + // Print results + print_results(&thread_data[i]); + + // Free allocated memory + if(thread_data[i].result != NULL) + free(thread_data[i].result); + } + return EXIT_SUCCESS; } diff --git a/src/tools/arp-scan.h b/src/tools/arp-scan.h index 90b5fec70..0a5b56711 100644 --- a/src/tools/arp-scan.h +++ b/src/tools/arp-scan.h @@ -11,6 +11,6 @@ #ifndef ARP_SCAN_H #define ARP_SCAN_H -int run_arp_scan(const bool verbose, const bool scan_all); +int run_arp_scan(const bool scan_all); #endif // ARP_SCAN_H From 5dcdbd7264b98e8d69269d60af1789779eb6477e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 May 2023 04:40:11 +0200 Subject: [PATCH 45/78] Scale progress percentage according to number of addresses to be scanned by the individual threads Signed-off-by: DL6ER --- src/tools/arp-scan.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 69e702412..b84ac4dd1 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -563,34 +563,35 @@ int run_arp_scan(const bool scan_all) while(!all_done) { all_done = true; - unsigned int num_scans = 0, total_scans = 0; + uint64_t num_scans = 0, total_scans = 0; for(unsigned int i = 0; i < tid; i++) { - if(thread_data[i].status == STATUS_SCANNING) + const uint32_t num_addresses = 1 << (32 - thread_data[i].dst_cidr); + if(thread_data[i].status == STATUS_INITIALIZING || + thread_data[i].status == STATUS_SCANNING) { // At least one thread is still scanning all_done = false; - num_scans += thread_data[i].num_scans; - total_scans += NUM_SCANS; } - if(thread_data[i].status == STATUS_COMPLETE) + if(thread_data[i].status == STATUS_SCANNING || + thread_data[i].status == STATUS_COMPLETE) { // Also add up scans for completed threads - num_scans += thread_data[i].num_scans; - total_scans += NUM_SCANS; + num_scans += thread_data[i].num_scans * num_addresses; + total_scans += NUM_SCANS * num_addresses; } } if(!all_done) { // Print progress - printf("%i%%... ", 100*num_scans/total_scans); + printf("%i%%... ", (int)(100*num_scans/total_scans)); // Flush stdout fflush(stdout); // Sleep for 1 second sleepms(1000); } } - puts("100%%\n\n"); + puts("100%\n\n"); // Wait for all threads to join back with us for(unsigned int i = 0; i < tid; i++) From 1bc05b9affc82c3ff9f6a7a054bc48576de36270 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 May 2023 04:45:39 +0200 Subject: [PATCH 46/78] Only print progress if it has changed. Otherwise, print "." as hearthbeat Signed-off-by: DL6ER --- src/tools/arp-scan.c | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index b84ac4dd1..8fb5351e3 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -94,13 +94,14 @@ struct thread_data { char ipstr[INET_ADDRSTRLEN]; unsigned char mac[16]; unsigned int num_scans; + uint32_t scanned_addresses; char *error; }; // Sends multiple ARP who-has request on interface ifindex, using source mac src_mac and source ip src_ip. // Iterates over all IP addresses in the range of dst_ip/cidr. static int send_arps(const int fd, const int ifindex, const char *iface, const unsigned char *src_mac, - struct in_addr *src_ip, struct in_addr dst_ip, const int dst_cidr) + struct in_addr *src_ip, struct in_addr dst_ip, const int dst_cidr, uint32_t *scanned_addresses) { int err = -1; unsigned char buffer[BUF_SIZE]; @@ -168,6 +169,8 @@ static int send_arps(const int fd, const int ifindex, const char *iface, const u // Increment IP address dst_ip.s_addr = htonl(ntohl(dst_ip.s_addr) + 1); + + (*scanned_addresses)++; } err = 0; @@ -395,14 +398,16 @@ static void *arp_scan_iface(void *args) // printf("Still scanning interface %s (%s/%i) %i%%...\n", iface, thread_data->ipstr, thread_data->dst_cidr, 100*scan_id/NUM_SCANS); // Send ARP requests to all IPs in subnet - if(send_arps(arp_socket, ifindex, iface, thread_data->mac, &thread_data->src_addr.sin_addr, thread_data->dst_addr.sin_addr, thread_data->dst_cidr) != 0) + if(send_arps(arp_socket, ifindex, iface, thread_data->mac, &thread_data->src_addr.sin_addr, + thread_data->dst_addr.sin_addr, thread_data->dst_cidr, &thread_data->scanned_addresses) != 0) { thread_data->status = STATUS_ERROR; break; } // Read ARP responses - if(read_arp(arp_socket, iface, &thread_data->dst_addr.sin_addr, thread_data->result, thread_data->result_size, thread_data->num_scans) != 0) + if(read_arp(arp_socket, iface, &thread_data->dst_addr.sin_addr, thread_data->result, thread_data->result_size, + thread_data->num_scans) != 0) { thread_data->status = STATUS_ERROR; break; @@ -560,13 +565,13 @@ int run_arp_scan(const bool scan_all) // Wait for all threads to finish scanning bool all_done = false; + unsigned int progress = 0; while(!all_done) { all_done = true; uint64_t num_scans = 0, total_scans = 0; for(unsigned int i = 0; i < tid; i++) { - const uint32_t num_addresses = 1 << (32 - thread_data[i].dst_cidr); if(thread_data[i].status == STATUS_INITIALIZING || thread_data[i].status == STATUS_SCANNING) { @@ -577,16 +582,29 @@ int run_arp_scan(const bool scan_all) thread_data[i].status == STATUS_COMPLETE) { // Also add up scans for completed threads - num_scans += thread_data[i].num_scans * num_addresses; - total_scans += NUM_SCANS * num_addresses; + num_scans += thread_data[i].scanned_addresses; + total_scans += NUM_SCANS * thread_data[i].result_size; } } if(!all_done) { - // Print progress - printf("%i%%... ", (int)(100*num_scans/total_scans)); + // Calculate progress (total number of scans / total number of addresses) + // We add 1 to total_scans to avoid division by zero + const unsigned int new_progress = 100 * num_scans / (total_scans + 1); + if(new_progress > progress) + { + // Print progress + printf(" %i%%", new_progress); + + // Update progress + progress = new_progress; + } + + putc('.', stdout); + // Flush stdout fflush(stdout); + // Sleep for 1 second sleepms(1000); } From 4387ff21ee8bf15f65adaf29f98b5b187354e370 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 May 2023 04:49:19 +0200 Subject: [PATCH 47/78] Always skip the loopback interface, also in "-a" mode Signed-off-by: DL6ER --- src/tools/arp-scan.c | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 8fb5351e3..1a038b3a8 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -346,10 +346,6 @@ static void *arp_scan_iface(void *args) // Set interface name as thread name prctl(PR_SET_NAME, iface, 0, 0, 0); - // Get interface IPv4 address - memcpy(&thread_data->src_addr, ifa->ifa_addr, sizeof(thread_data->src_addr)); - inet_ntop(AF_INET, &thread_data->src_addr.sin_addr, thread_data->ipstr, INET_ADDRSTRLEN); - // Get interface netmask memcpy(&thread_data->mask, ifa->ifa_netmask, sizeof(thread_data->mask)); @@ -547,16 +543,24 @@ int run_arp_scan(const bool scan_all) { thread_data[tid].ifa = tmp; thread_data[tid].iface = tmp->ifa_name; - // Create thread - if(pthread_create(&scanthread[tid], &attr, arp_scan_iface, &thread_data[tid] ) != 0) + + // Get interface IPv4 address + memcpy(&thread_data[tid].src_addr, tmp->ifa_addr, sizeof(thread_data[tid].src_addr)); + inet_ntop(AF_INET, &thread_data[tid].src_addr.sin_addr, thread_data[tid].ipstr, INET_ADDRSTRLEN); + + // Always skip the loopback interface + if(thread_data[tid].src_addr.sin_addr.s_addr != htonl(INADDR_LOOPBACK)) { - printf("Unable to launch thread for interface %s, skipping...\n", - tmp->ifa_name); - continue; + // Create thread + if(pthread_create(&scanthread[tid], &attr, arp_scan_iface, &thread_data[tid] ) != 0) + { + printf("Unable to launch thread for interface %s, skipping...\n", + tmp->ifa_name); + } + + // Increase thread ID + tid++; } - - // Increase thread ID - tid++; } // Advance to the next interface From d45a7c46fc23c32bed0906207be7efc7dced8a0b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 May 2023 04:51:01 +0200 Subject: [PATCH 48/78] Interface names can be up to 16 bytes long. Docker bridge interfaces actually use this space so we need to reserve enough space here Signed-off-by: DL6ER --- src/tools/arp-scan.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 1a038b3a8..1c446840f 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -448,7 +448,7 @@ static void print_results(struct thread_data *thread_data) // If there is at least one result, print header printf("ARP scan on interface %s (%s/%i) finished\n", thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); - printf("%-16s %-10s %-24s %-17s Reply matrix\n", + printf("%-16s %-16s %-24s %-17s Reply matrix\n", "IP address", "Interface", "Hostname", "MAC address"); // Add our own IP address to the results so IP conflicts can be detected @@ -487,7 +487,7 @@ static void print_results(struct thread_data *thread_data) break; // Print MAC address - printf("%-16s %-10s %-24s %02x:%02x:%02x:%02x:%02x:%02x ", + printf("%-16s %-16s %-24s %02x:%02x:%02x:%02x:%02x:%02x ", thread_data->ipstr, thread_data->iface, get_hostname(&ip), thread_data->result[i].device[j].mac[0], From 2d6a3619a07fb5a41ba95d232912ded1d0149c6b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 May 2023 11:49:11 +0200 Subject: [PATCH 49/78] Clearly log when scanning interfaces failed Signed-off-by: DL6ER --- src/tools/arp-scan.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 1c446840f..60b60d700 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -430,6 +430,13 @@ static void print_results(struct thread_data *thread_data) return; } + if(thread_data->status == STATUS_ERROR) + { + printf("Error scanning interface %s (%s/%i)\n\n", + thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); + return; + } + // Check if there are any results unsigned int replies = 0; for(unsigned int i = 0; i < thread_data->result_size; i++) @@ -440,7 +447,7 @@ static void print_results(struct thread_data *thread_data) // Exit early if there are no results if(replies == 0) { - printf("No devices found on interface %s (%s/%i)\n\n", + printf("No devices replied on interface %s (%s/%i)\n\n", thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); return; } From 8a299112b83a9527dac385fc1106aaf41814ca4c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 May 2023 20:54:51 +0200 Subject: [PATCH 50/78] Log more verbose human-readable error string if available Signed-off-by: DL6ER --- src/tools/arp-scan.c | 53 +++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 60b60d700..c3eef0e81 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -95,12 +95,12 @@ struct thread_data { unsigned char mac[16]; unsigned int num_scans; uint32_t scanned_addresses; - char *error; + const char *error; }; // Sends multiple ARP who-has request on interface ifindex, using source mac src_mac and source ip src_ip. // Iterates over all IP addresses in the range of dst_ip/cidr. -static int send_arps(const int fd, const int ifindex, const char *iface, const unsigned char *src_mac, +static int send_arps(const int fd, const int ifindex, const char *iface, const unsigned char *src_mac, const char **error, struct in_addr *src_ip, struct in_addr dst_ip, const int dst_cidr, uint32_t *scanned_addresses) { int err = -1; @@ -161,9 +161,8 @@ static int send_arps(const int fd, const int ifindex, const char *iface, const u ret = sendto(fd, buffer, 42, 0, (struct sockaddr *) &socket_address, sizeof(socket_address)); if (ret == -1) { - if(errno != EPROTONOSUPPORT) - printf("Unable to send ARP request for %s@%s: %s\n", - inet_ntoa(dst_ip), iface, strerror(errno)); + err = errno; + *error = strerror(err); goto out; } @@ -178,13 +177,14 @@ static int send_arps(const int fd, const int ifindex, const char *iface, const u return err; } -static int create_arp_socket(const int ifindex, const char *iface) +static int create_arp_socket(const int ifindex, const char *iface, const char **error) { // Create socket for ARP communications const int arp_socket = socket(AF_PACKET, SOCK_RAW, htons(ETH_P_ARP)); if(arp_socket < 0) { - printf("Unable to create socket for ARP communications on interface %s: %s\n", iface, strerror(errno)); + *error = strerror(errno); + printf("Unable to create socket for ARP communications on interface %s: %s\n", iface, *error); return -1; } @@ -195,7 +195,8 @@ static int create_arp_socket(const int ifindex, const char *iface) sll.sll_ifindex = ifindex; if (bind(arp_socket, (struct sockaddr*) &sll, sizeof(struct sockaddr_ll)) < 0) { - printf("Unable to bind socket for ARP communications on interface %s: %s\n", iface, strerror(errno)); + *error = strerror(errno); + printf("Unable to bind socket for ARP communications on interface %s: %s\n", iface, *error); close(arp_socket); return -1; } @@ -206,7 +207,8 @@ static int create_arp_socket(const int ifindex, const char *iface) tv.tv_usec = 0; if (setsockopt(arp_socket, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0) { - printf("Unable to set timeout for ARP communications on interface %s: %s\n", iface, strerror(errno)); + *error = strerror(errno); + printf("Unable to set timeout for ARP communications on interface %s: %s\n", iface, *error); close(arp_socket); return -1; } @@ -249,7 +251,7 @@ static void add_result(const char *iface, struct in_addr *rcv_ip, struct in_addr } // Read all ARP responses -static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, +static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, const char **error, struct arp_result *result, const size_t result_len, const unsigned int scan_id) { ssize_t ret = 0; @@ -269,7 +271,8 @@ static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, } // Error - printf("recvfrom(): %s", strerror(errno)); + *error = strerror(errno); + printf("recvfrom(): %s", *error); break; } struct ethhdr *rcv_resp = (struct ethhdr *) buffer; @@ -359,15 +362,18 @@ static void *arp_scan_iface(void *args) if(thread_data->dst_cidr < 24 && !arp_all) { thread_data->status = STATUS_SKIPPED_CIDR_MISMATCH; - //printf("Skipped interface %s (%s/%i)\n", iface, thread_data->ipstr, thread_data->dst_cidr); +#ifdef DEBUG + printf("Skipped interface %s (%s/%i)\n", iface, thread_data->ipstr, thread_data->dst_cidr); +#endif pthread_exit(NULL); } - //if(arp_verbose) - // printf("Scanning interface %s (%s/%i)...\n", iface, thread_data->ipstr, thread_data->dst_cidr); +#ifdef DEBUG + printf("Scanning interface %s (%s/%i)...\n", iface, thread_data->ipstr, thread_data->dst_cidr); +#endif thread_data->status = STATUS_SCANNING; // Create socket for ARP communications - const int arp_socket = create_arp_socket(ifindex, iface); + const int arp_socket = create_arp_socket(ifindex, iface, &thread_data->error); // Cannot create socket, likely a permission error if(arp_socket < 0) @@ -390,11 +396,11 @@ static void *arp_scan_iface(void *args) for(thread_data->num_scans = 0; thread_data->num_scans < NUM_SCANS; thread_data->num_scans++) { - //if(arp_verbose) - // printf("Still scanning interface %s (%s/%i) %i%%...\n", iface, thread_data->ipstr, thread_data->dst_cidr, 100*scan_id/NUM_SCANS); - +#ifdef DEBUG + printf("Still scanning interface %s (%s/%i) %i%%...\n", iface, thread_data->ipstr, thread_data->dst_cidr, 100*scan_id/NUM_SCANS); +#endif // Send ARP requests to all IPs in subnet - if(send_arps(arp_socket, ifindex, iface, thread_data->mac, &thread_data->src_addr.sin_addr, + if(send_arps(arp_socket, ifindex, iface, thread_data->mac, &thread_data->error, &thread_data->src_addr.sin_addr, thread_data->dst_addr.sin_addr, thread_data->dst_cidr, &thread_data->scanned_addresses) != 0) { thread_data->status = STATUS_ERROR; @@ -402,8 +408,8 @@ static void *arp_scan_iface(void *args) } // Read ARP responses - if(read_arp(arp_socket, iface, &thread_data->dst_addr.sin_addr, thread_data->result, thread_data->result_size, - thread_data->num_scans) != 0) + if(read_arp(arp_socket, iface, &thread_data->dst_addr.sin_addr, &thread_data->error, + thread_data->result, thread_data->result_size, thread_data->num_scans) != 0) { thread_data->status = STATUS_ERROR; break; @@ -432,8 +438,9 @@ static void print_results(struct thread_data *thread_data) if(thread_data->status == STATUS_ERROR) { - printf("Error scanning interface %s (%s/%i)\n\n", - thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); + printf("Error scanning interface %s (%s/%i)%s%s\n\n", + thread_data->iface, thread_data->ipstr, thread_data->dst_cidr, + thread_data->error ? ": " : "", thread_data->error ? thread_data->error : ""); return; } From e3550c1a472d1bcdf624277edf66e1dcce86b94f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 May 2023 21:02:29 +0200 Subject: [PATCH 51/78] Add capabilities check for CAP_NET_RAW (root always has it) Signed-off-by: DL6ER --- src/capabilities.c | 41 +++++++++++++++++++++++++++++++++++++++++ src/capabilities.h | 1 + src/tools/arp-scan.c | 16 ++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/src/capabilities.c b/src/capabilities.c index 603cb4f76..49e5d5d96 100644 --- a/src/capabilities.c +++ b/src/capabilities.c @@ -21,6 +21,47 @@ static const unsigned int capabilityIDs[] = { CAP_CHOWN , CAP_DAC_OVERRIDE , static const char* capabilityNames[] = {"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", "CAP_FSETID", "CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", "CAP_LINUX_IMMUTABLE", "CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", "CAP_NET_ADMIN", "CAP_NET_RAW", "CAP_IPC_LOCK", "CAP_IPC_OWNER", "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT", "CAP_SYS_PTRACE", "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE", "CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", "CAP_LEASE", "CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP"}; static const unsigned int numCaps = sizeof(capabilityIDs) / sizeof(*capabilityIDs); +bool check_capability(const unsigned int cap) +{ + // First assume header version 1 + int capsize = 1; // VFS_CAP_U32_1 + cap_user_data_t data = NULL; + cap_user_header_t hdr = calloc(sizeof(*hdr), capsize); + + // Determine capabilities version used by the current kernel + capget(hdr, NULL); + + // Check version + if (hdr->version != LINUX_CAPABILITY_VERSION_1) + { + // If unknown version, use largest supported version (3) + // Version 2 is deprecated according to linux/capability.h + if (hdr->version != LINUX_CAPABILITY_VERSION_2) + { + hdr->version = LINUX_CAPABILITY_VERSION_3; + capsize = 2; // VFS_CAP_U32_3 + } + else + { + // Use version 2 + capsize = 2; // VFS_CAP_U32_2 + } + } + + // Get current capabilities + data = calloc(sizeof(*data), capsize); + capget(hdr, data); + + // Check if the capability is available + const bool available = ((data->permitted & (1 << cap)) && (data->effective & (1 << cap))); + + // Free memory + free(hdr); + free(data); + + return available; +} + bool check_capabilities(void) { // First assume header version 1 diff --git a/src/capabilities.h b/src/capabilities.h index a65113828..6f811301e 100644 --- a/src/capabilities.h +++ b/src/capabilities.h @@ -10,6 +10,7 @@ #ifndef CAPABILITIES_H #define CAPABILITIES_H +bool check_capability(const unsigned int cap); bool check_capabilities(void); #endif //CAPABILITIES_H diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index c3eef0e81..964c07db5 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -16,6 +16,9 @@ #include "dhcp-discover.h" // sleepms() #include "timers.h" +// check_capability() +#include "capabilities.h" +#include #include #include @@ -184,7 +187,9 @@ static int create_arp_socket(const int ifindex, const char *iface, const char ** if(arp_socket < 0) { *error = strerror(errno); +#ifdef DEBUG printf("Unable to create socket for ARP communications on interface %s: %s\n", iface, *error); +#endif return -1; } @@ -196,7 +201,9 @@ static int create_arp_socket(const int ifindex, const char *iface, const char ** if (bind(arp_socket, (struct sockaddr*) &sll, sizeof(struct sockaddr_ll)) < 0) { *error = strerror(errno); +#ifdef DEBUG printf("Unable to bind socket for ARP communications on interface %s: %s\n", iface, *error); +#endif close(arp_socket); return -1; } @@ -208,7 +215,9 @@ static int create_arp_socket(const int ifindex, const char *iface, const char ** if (setsockopt(arp_socket, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)) < 0) { *error = strerror(errno); +#ifdef DEBUG printf("Unable to set timeout for ARP communications on interface %s: %s\n", iface, *error); +#endif close(arp_socket); return -1; } @@ -530,6 +539,13 @@ static void print_results(struct thread_data *thread_data) int run_arp_scan(const bool scan_all) { + // Check if we are capable of sending ARP packets + if(!check_capability(CAP_NET_RAW)) + { + puts("Error: Insufficient permissions or capabilities (needs CAP_NET_RAW). Try running as root (sudo)"); + return EXIT_FAILURE; + } + arp_all = scan_all; puts("Discovering IPv4 hosts on the network using the Address Resolution Protocol (ARP)...\n"); From 73e3d8840864270bf4d3fcd3244c965045d9ad91 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 17 May 2023 16:53:51 +0200 Subject: [PATCH 52/78] Optimize thread_data structure and store a thread-local copy of the interface name Signed-off-by: DL6ER --- src/tools/arp-scan.c | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 964c07db5..3a81c1ebe 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -82,23 +82,23 @@ enum status { STATUS_SCANNING, STATUS_ERROR, STATUS_COMPLETE -}; +} __attribute__ ((packed)); struct thread_data { - int dst_cidr; - struct sockaddr_in src_addr; - struct sockaddr_in dst_addr; - struct sockaddr_in mask; - struct ifaddrs *ifa; - const char *iface; - struct arp_result *result; - size_t result_size; - enum status status; + char iface[IF_NAMESIZE + 1]; char ipstr[INET_ADDRSTRLEN]; - unsigned char mac[16]; + unsigned char mac[MAC_LENGTH]; + enum status status; + int dst_cidr; unsigned int num_scans; + size_t result_size; uint32_t scanned_addresses; const char *error; + struct ifaddrs *ifa; + struct arp_result *result; + struct sockaddr_in src_addr; + struct sockaddr_in dst_addr; + struct sockaddr_in mask; }; // Sends multiple ARP who-has request on interface ifindex, using source mac src_mac and source ip src_ip. @@ -488,11 +488,17 @@ static void print_results(struct thread_data *thread_data) // Print MAC addresses for(j = 0; j < MAX_MACS; j++) { + // Check if result[i].mac[j] is all-zero, if so, skip this entry + if(memcmp(thread_data->result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", 6) == 0) + break; + // Check if IP address replied + replies = 0u; bool replied = false; for(unsigned int k = 0; k < NUM_SCANS; k++) { replied |= thread_data->result[i].device[j].replied[k] > 0; + replies += thread_data->result[i].device[j].replied[k] > 0 ? 1 : 0; multiple_replies += thread_data->result[i].device[j].replied[k] > 1; } if(!replied) @@ -505,9 +511,6 @@ static void print_results(struct thread_data *thread_data) struct in_addr ip = { 0 }; ip.s_addr = htonl(ntohl(thread_data->dst_addr.sin_addr.s_addr) + i); inet_ntop(AF_INET, &ip, thread_data->ipstr, INET_ADDRSTRLEN); - // Check if result[i].mac[j] is all-zero - if(memcmp(thread_data->result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", 6) == 0) - break; // Print MAC address printf("%-16s %-16s %-24s %02x:%02x:%02x:%02x:%02x:%02x ", @@ -523,7 +526,7 @@ static void print_results(struct thread_data *thread_data) for(unsigned int k = 0; k < NUM_SCANS; k++) printf(" %s", thread_data->result[i].device[j].replied[k] > 0 ? "X" : "-"); - putc('\n', stdout); + printf(" (%u%%)\n", replies * 100 / NUM_SCANS); } // Print warning if we received multiple replies @@ -572,7 +575,7 @@ int run_arp_scan(const bool scan_all) if(tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_INET) { thread_data[tid].ifa = tmp; - thread_data[tid].iface = tmp->ifa_name; + strncpy(thread_data[tid].iface, tmp->ifa_name, sizeof(thread_data[tid].iface) - 1); // Get interface IPv4 address memcpy(&thread_data[tid].src_addr, tmp->ifa_addr, sizeof(thread_data[tid].src_addr)); @@ -643,7 +646,7 @@ int run_arp_scan(const bool scan_all) sleepms(1000); } } - puts("100%\n\n"); + puts("100%\n"); // Wait for all threads to join back with us for(unsigned int i = 0; i < tid; i++) From 6ce6eaeda512ccde3325b3e572667080ba0987aa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 17 May 2023 17:02:15 +0200 Subject: [PATCH 53/78] Add arp-scan -xtreme mode for very unreliable connections Signed-off-by: DL6ER --- src/args.c | 9 ++++++--- src/tools/arp-scan.c | 34 ++++++++++++++++++---------------- src/tools/arp-scan.h | 2 +- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/args.c b/src/args.c index 14b4031d5..16391ff4f 100644 --- a/src/args.c +++ b/src/args.c @@ -178,8 +178,9 @@ void parse_args(int argc, char* argv[]) { // Enable stdout printing cli_mode = true; - const bool arp_all = argc > 2 && strcmp(argv[2], "-a") == 0; - exit(run_arp_scan(arp_all)); + const bool scan_all = argc > 2 && strcmp(argv[2], "-a") == 0; + const bool extreme_mode = argc > 2 && strcmp(argv[2], "-x") == 0; + exit(run_arp_scan(scan_all, extreme_mode)); } // start from 1, as argv[0] is the executable name @@ -513,10 +514,12 @@ void parse_args(int argc, char* argv[]) printf("%sOther:%s\n", yellow, normal); printf("\t%sdhcp-discover%s Discover DHCP servers in the local\n", green, normal); printf("\t network\n"); - printf("\t%sarp-scan %s[-a]%s Use ARP to scan local network for\n", green, cyan, normal); + printf("\t%sarp-scan %s[-a/-x]%s Use ARP to scan local network for\n", green, cyan, normal); printf("\t possible IP conflicts\n"); printf("\t Append %s-a%s to force scan on all\n", cyan, normal); printf("\t interfaces\n"); + printf("\t Append %s-x%s to force scan on all\n", cyan, normal); + printf("\t interfaces and scan 10x more often\n"); printf("\t%s-h%s, %shelp%s Display this help and exit\n\n", green, normal, green, normal); exit(EXIT_SUCCESS); } diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 3a81c1ebe..408ae6392 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -40,9 +40,6 @@ // How long do we wait for ARP replies in each scan [seconds]? #define ARP_TIMEOUT 1 -// Global constant -static bool arp_all = false; - // Protocol definitions #define PROTO_ARP 0x0806 #define ETH2_HEADER_LEN 14 @@ -71,7 +68,8 @@ struct arp_header { struct arp_result { struct device { - unsigned int replied[NUM_SCANS]; + // In extreme mode, we can scan up to 10x more often + unsigned int replied[10*NUM_SCANS]; unsigned char mac[MAC_LENGTH]; } device[MAX_MACS]; }; @@ -85,12 +83,14 @@ enum status { } __attribute__ ((packed)); struct thread_data { + bool scan_all :1; char iface[IF_NAMESIZE + 1]; char ipstr[INET_ADDRSTRLEN]; unsigned char mac[MAC_LENGTH]; enum status status; int dst_cidr; unsigned int num_scans; + unsigned int total_scans; size_t result_size; uint32_t scanned_addresses; const char *error; @@ -368,7 +368,7 @@ static void *arp_scan_iface(void *args) const int ifindex = if_nametoindex(iface); // Scan only interfaces with CIDR >= 24 - if(thread_data->dst_cidr < 24 && !arp_all) + if(thread_data->dst_cidr < 24 && !thread_data->scan_all) { thread_data->status = STATUS_SKIPPED_CIDR_MISMATCH; #ifdef DEBUG @@ -403,10 +403,10 @@ static void *arp_scan_iface(void *args) struct arp_result *result = calloc(thread_data->result_size, sizeof(struct arp_result)); thread_data->result = result; - for(thread_data->num_scans = 0; thread_data->num_scans < NUM_SCANS; thread_data->num_scans++) + for(thread_data->num_scans = 0; thread_data->num_scans < thread_data->total_scans; thread_data->num_scans++) { #ifdef DEBUG - printf("Still scanning interface %s (%s/%i) %i%%...\n", iface, thread_data->ipstr, thread_data->dst_cidr, 100*scan_id/NUM_SCANS); + printf("Still scanning interface %s (%s/%i) %i%%...\n", iface, thread_data->ipstr, thread_data->dst_cidr, 100*scan_id/thread_data->total_scans); #endif // Send ARP requests to all IPs in subnet if(send_arps(arp_socket, ifindex, iface, thread_data->mac, &thread_data->error, &thread_data->src_addr.sin_addr, @@ -440,7 +440,7 @@ static void print_results(struct thread_data *thread_data) if(thread_data->status == STATUS_SKIPPED_CIDR_MISMATCH) { - printf("Skipped interface %s (%s/%i) because of too large network (use -a to force scanning this interface)\n\n", + printf("Skipped interface %s (%s/%i) because of too large network (use -a or -x to force scanning this interface)\n\n", thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); return; } @@ -457,7 +457,7 @@ static void print_results(struct thread_data *thread_data) unsigned int replies = 0; for(unsigned int i = 0; i < thread_data->result_size; i++) for(unsigned int j = 0; j < MAX_MACS; j++) - for(unsigned int k = 0; k < NUM_SCANS; k++) + for(unsigned int k = 0; k < thread_data->total_scans; k++) replies += thread_data->result[i].device[j].replied[k]; // Exit early if there are no results @@ -476,7 +476,7 @@ static void print_results(struct thread_data *thread_data) // Add our own IP address to the results so IP conflicts can be detected // (our own IP address is not included in the ARP scan) - for(unsigned int i = 0; i < NUM_SCANS; i++) + for(unsigned int i = 0; i < thread_data->total_scans; i++) add_result(thread_data->iface, &thread_data->src_addr.sin_addr, &thread_data->dst_addr.sin_addr, thread_data->mac, thread_data->result, thread_data->result_size, i); // Print results @@ -495,7 +495,7 @@ static void print_results(struct thread_data *thread_data) // Check if IP address replied replies = 0u; bool replied = false; - for(unsigned int k = 0; k < NUM_SCANS; k++) + for(unsigned int k = 0; k < thread_data->total_scans; k++) { replied |= thread_data->result[i].device[j].replied[k] > 0; replies += thread_data->result[i].device[j].replied[k] > 0 ? 1 : 0; @@ -523,10 +523,10 @@ static void print_results(struct thread_data *thread_data) thread_data->result[i].device[j].mac[4], thread_data->result[i].device[j].mac[5]); - for(unsigned int k = 0; k < NUM_SCANS; k++) + for(unsigned int k = 0; k < thread_data->total_scans; k++) printf(" %s", thread_data->result[i].device[j].replied[k] > 0 ? "X" : "-"); - printf(" (%u%%)\n", replies * 100 / NUM_SCANS); + printf(" (%u%%)\n", replies * 100 / thread_data->total_scans); } // Print warning if we received multiple replies @@ -540,7 +540,7 @@ static void print_results(struct thread_data *thread_data) putc('\n', stdout); } -int run_arp_scan(const bool scan_all) +int run_arp_scan(const bool scan_all, const bool extreme_mode) { // Check if we are capable of sending ARP packets if(!check_capability(CAP_NET_RAW)) @@ -549,7 +549,6 @@ int run_arp_scan(const bool scan_all) return EXIT_FAILURE; } - arp_all = scan_all; puts("Discovering IPv4 hosts on the network using the Address Resolution Protocol (ARP)...\n"); // Get interface names for available interfaces on this machine @@ -581,6 +580,9 @@ int run_arp_scan(const bool scan_all) memcpy(&thread_data[tid].src_addr, tmp->ifa_addr, sizeof(thread_data[tid].src_addr)); inet_ntop(AF_INET, &thread_data[tid].src_addr.sin_addr, thread_data[tid].ipstr, INET_ADDRSTRLEN); + thread_data[tid].scan_all = scan_all || extreme_mode; + thread_data[tid].total_scans = extreme_mode ? 10*NUM_SCANS : NUM_SCANS; + // Always skip the loopback interface if(thread_data[tid].src_addr.sin_addr.s_addr != htonl(INADDR_LOOPBACK)) { @@ -620,7 +622,7 @@ int run_arp_scan(const bool scan_all) { // Also add up scans for completed threads num_scans += thread_data[i].scanned_addresses; - total_scans += NUM_SCANS * thread_data[i].result_size; + total_scans += thread_data[i].total_scans * thread_data[i].result_size; } } if(!all_done) diff --git a/src/tools/arp-scan.h b/src/tools/arp-scan.h index 0a5b56711..c22eafd44 100644 --- a/src/tools/arp-scan.h +++ b/src/tools/arp-scan.h @@ -11,6 +11,6 @@ #ifndef ARP_SCAN_H #define ARP_SCAN_H -int run_arp_scan(const bool scan_all); +int run_arp_scan(const bool scan_all, const bool extreme_mode); #endif // ARP_SCAN_H From ba8807cc6e4e61c3de8794e661bbebaed852b5bc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 17 May 2023 22:24:53 +0200 Subject: [PATCH 54/78] Give reply rate in percent instead of showing the reply matrix Signed-off-by: DL6ER --- src/tools/arp-scan.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 408ae6392..3fa9ec2ec 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -471,8 +471,8 @@ static void print_results(struct thread_data *thread_data) // If there is at least one result, print header printf("ARP scan on interface %s (%s/%i) finished\n", thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); - printf("%-16s %-16s %-24s %-17s Reply matrix\n", - "IP address", "Interface", "Hostname", "MAC address"); + printf("%-16s %-16s %-24s %-17s %s\n", + "IP address", "Interface", "Hostname", "MAC address", "Reply rate"); // Add our own IP address to the results so IP conflicts can be detected // (our own IP address is not included in the ARP scan) @@ -483,7 +483,6 @@ static void print_results(struct thread_data *thread_data) for(unsigned int i = 0; i < thread_data->result_size; i++) { unsigned int j = 0, replied_devices = 0; - unsigned int multiple_replies = 0; // Print MAC addresses for(j = 0; j < MAX_MACS; j++) @@ -495,6 +494,7 @@ static void print_results(struct thread_data *thread_data) // Check if IP address replied replies = 0u; bool replied = false; + unsigned int multiple_replies = 0; for(unsigned int k = 0; k < thread_data->total_scans; k++) { replied |= thread_data->result[i].device[j].replied[k] > 0; @@ -513,7 +513,7 @@ static void print_results(struct thread_data *thread_data) inet_ntop(AF_INET, &ip, thread_data->ipstr, INET_ADDRSTRLEN); // Print MAC address - printf("%-16s %-16s %-24s %02x:%02x:%02x:%02x:%02x:%02x ", + printf("%-16s %-16s %-24s %02x:%02x:%02x:%02x:%02x:%02x %u %%\n", thread_data->ipstr, thread_data->iface, get_hostname(&ip), thread_data->result[i].device[j].mac[0], @@ -521,21 +521,29 @@ static void print_results(struct thread_data *thread_data) thread_data->result[i].device[j].mac[2], thread_data->result[i].device[j].mac[3], thread_data->result[i].device[j].mac[4], - thread_data->result[i].device[j].mac[5]); + thread_data->result[i].device[j].mac[5], + replies * 100 / thread_data->total_scans); +#ifdef DEBUG for(unsigned int k = 0; k < thread_data->total_scans; k++) printf(" %s", thread_data->result[i].device[j].replied[k] > 0 ? "X" : "-"); - - printf(" (%u%%)\n", replies * 100 / thread_data->total_scans); +#endif + if(multiple_replies > 0) + printf("WARNING: Received multiple replies from %02x:%02x:%02x:%02x:%02x:%02x for %s in %i scan%s\n", + thread_data->result[i].device[j].mac[0], + thread_data->result[i].device[j].mac[1], + thread_data->result[i].device[j].mac[2], + thread_data->result[i].device[j].mac[3], + thread_data->result[i].device[j].mac[4], + thread_data->result[i].device[j].mac[5], + thread_data->ipstr, + multiple_replies, multiple_replies > 1 ? "s" : ""); } // Print warning if we received multiple replies if(replied_devices > 1) printf("WARNING: Received replies for %s from %i devices\n", thread_data->ipstr, replied_devices); - if(multiple_replies > 0) - printf("WARNING: Received multiple replies for %s in %i scan%s\n", - thread_data->ipstr, multiple_replies, multiple_replies > 1 ? "s" : ""); } putc('\n', stdout); } @@ -633,7 +641,7 @@ int run_arp_scan(const bool scan_all, const bool extreme_mode) if(new_progress > progress) { // Print progress - printf(" %i%%", new_progress); + printf(" %i%% ", new_progress); // Update progress progress = new_progress; From 615be9c2c9ba60e95378c6c727996fe48687e3c6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 May 2023 09:04:26 +0200 Subject: [PATCH 55/78] Exit early if insufficient memory is available, perform as many interface scans as possible under these conditions Signed-off-by: DL6ER --- src/tools/arp-scan.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 3fa9ec2ec..57bd648b5 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -399,8 +399,18 @@ static void *arp_scan_iface(void *args) // Allocate memory for ARP response buffer const size_t arp_result_len = 1 << (32 - thread_data->dst_cidr); + struct arp_result *result = calloc(arp_result_len, sizeof(struct arp_result)); + if(result == NULL) + { + // Memory allocation failed due to insufficient memory being + // available + thread_data->status = STATUS_ERROR; + thread_data->error = strerror(ENOMEM); + pthread_exit(NULL); + } + + // Store ARP response buffer in thread_data thread_data->result_size = arp_result_len; - struct arp_result *result = calloc(thread_data->result_size, sizeof(struct arp_result)); thread_data->result = result; for(thread_data->num_scans = 0; thread_data->num_scans < thread_data->total_scans; thread_data->num_scans++) From 4d7640c9bed607dd2888c41a19ff2e9867328e27 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 May 2023 09:08:26 +0200 Subject: [PATCH 56/78] Reduce memory requirements by factor 4x Signed-off-by: DL6ER --- src/tools/arp-scan.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 57bd648b5..4442e8222 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -69,11 +69,13 @@ struct arp_header { struct arp_result { struct device { // In extreme mode, we can scan up to 10x more often - unsigned int replied[10*NUM_SCANS]; + unsigned char replied[10*NUM_SCANS]; unsigned char mac[MAC_LENGTH]; } device[MAX_MACS]; }; +const size_t arp_result_size = sizeof(struct arp_result); + enum status { STATUS_INITIALIZING = 0, STATUS_SKIPPED_CIDR_MISMATCH, From d1f70d7d6c1b89266f11a0ac666f8e09ae2b4b33 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 May 2023 09:45:24 +0200 Subject: [PATCH 57/78] Further reduce memory requirements by factor 10x (if not in -x mode) Signed-off-by: DL6ER --- src/tools/arp-scan.c | 182 ++++++++++++++++++++++++++----------------- 1 file changed, 111 insertions(+), 71 deletions(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 4442e8222..14f9c9f01 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -68,14 +68,19 @@ struct arp_header { struct arp_result { struct device { + unsigned char replied[NUM_SCANS]; + unsigned char mac[MAC_LENGTH]; + } device[MAX_MACS]; +}; + +struct arp_result_extreme { + struct device_extreme { // In extreme mode, we can scan up to 10x more often unsigned char replied[10*NUM_SCANS]; unsigned char mac[MAC_LENGTH]; } device[MAX_MACS]; }; -const size_t arp_result_size = sizeof(struct arp_result); - enum status { STATUS_INITIALIZING = 0, STATUS_SKIPPED_CIDR_MISMATCH, @@ -86,6 +91,7 @@ enum status { struct thread_data { bool scan_all :1; + bool extreme :1; char iface[IF_NAMESIZE + 1]; char ipstr[INET_ADDRSTRLEN]; unsigned char mac[MAC_LENGTH]; @@ -97,7 +103,10 @@ struct thread_data { uint32_t scanned_addresses; const char *error; struct ifaddrs *ifa; - struct arp_result *result; + union { + struct arp_result_extreme *result_extreme; + struct arp_result *result; + }; struct sockaddr_in src_addr; struct sockaddr_in dst_addr; struct sockaddr_in mask; @@ -105,8 +114,7 @@ struct thread_data { // Sends multiple ARP who-has request on interface ifindex, using source mac src_mac and source ip src_ip. // Iterates over all IP addresses in the range of dst_ip/cidr. -static int send_arps(const int fd, const int ifindex, const char *iface, const unsigned char *src_mac, const char **error, - struct in_addr *src_ip, struct in_addr dst_ip, const int dst_cidr, uint32_t *scanned_addresses) +static int send_arps(const int fd, const int ifindex, struct thread_data *thread_data) { int err = -1; unsigned char buffer[BUF_SIZE]; @@ -134,9 +142,9 @@ static int send_arps(const int fd, const int ifindex, const char *iface, const u memset(arp_req->target_mac, 0x00, MAC_LENGTH); // Source MAC to our own MAC address - memcpy(send_req->h_source, src_mac, MAC_LENGTH); - memcpy(arp_req->sender_mac, src_mac, MAC_LENGTH); - memcpy(socket_address.sll_addr, src_mac, MAC_LENGTH); + memcpy(send_req->h_source, thread_data->mac, MAC_LENGTH); + memcpy(arp_req->sender_mac, thread_data->mac, MAC_LENGTH); + memcpy(socket_address.sll_addr, thread_data->mac, MAC_LENGTH); // Protocol type is ARP send_req->h_proto = htons(ETH_P_ARP); @@ -149,11 +157,12 @@ static int send_arps(const int fd, const int ifindex, const char *iface, const u arp_req->opcode = htons(ARP_REQUEST); // Copy IP address to arp_req - memcpy(arp_req->sender_ip, &src_ip->s_addr, sizeof(src_ip->s_addr)); + memcpy(arp_req->sender_ip, &thread_data->src_addr.sin_addr.s_addr, sizeof(thread_data->src_addr.sin_addr.s_addr)); // Loop over all possible IP addresses in the range dst_ip/cidr // We start at 1 because the first IP address has already been set above - for(unsigned int i = 0; i < (1u << (32 - dst_cidr)); i++) + struct in_addr dst_ip = thread_data->dst_addr.sin_addr; + for(unsigned int i = 0; i < (1u << (32 - thread_data->dst_cidr)); i++) { // Fill in target IP address memcpy(arp_req->target_ip, &dst_ip.s_addr, sizeof(dst_ip.s_addr)); @@ -167,14 +176,14 @@ static int send_arps(const int fd, const int ifindex, const char *iface, const u if (ret == -1) { err = errno; - *error = strerror(err); + thread_data->error = strerror(err); goto out; } // Increment IP address dst_ip.s_addr = htonl(ntohl(dst_ip.s_addr) + 1); - (*scanned_addresses)++; + thread_data->scanned_addresses++; } err = 0; @@ -227,15 +236,16 @@ static int create_arp_socket(const int ifindex, const char *iface, const char ** return arp_socket; } -static void add_result(const char *iface, struct in_addr *rcv_ip, struct in_addr *dst_ip, unsigned char *sender_mac, - struct arp_result *result, const size_t result_len, const unsigned int scan_id) +static void add_result(struct in_addr *rcv_ip, unsigned char *sender_mac, + struct thread_data *thread_data, const unsigned int scan_id) { // Check if we have already found this IP address - uint32_t i = ntohl(rcv_ip->s_addr) - ntohl(dst_ip->s_addr); - if(i >= result_len) + uint32_t i = ntohl(rcv_ip->s_addr) - ntohl(thread_data->dst_addr.sin_addr.s_addr); + if(i >= thread_data->result_size) { - printf("Received IP address %s out of range for interface %s (%u >= %zu)\n", inet_ntoa(*rcv_ip), iface, i, result_len); + printf("Received IP address %s out of range for interface %s (%u >= %zu)\n", + inet_ntoa(*rcv_ip), thread_data->iface, i, thread_data->result_size); return; } @@ -243,27 +253,31 @@ static void add_result(const char *iface, struct in_addr *rcv_ip, struct in_addr unsigned int j = 0; for(; j < MAX_MACS; j++) { + unsigned char *mac = thread_data->extreme ? + thread_data->result_extreme[i].device[j].mac : + thread_data->result[i].device[j].mac; // Check if received MAC is already stored in result[i].device[j].mac - if(memcmp(result[i].device[j].mac, sender_mac, MAC_LENGTH) == 0) + if(memcmp(mac, sender_mac, MAC_LENGTH) == 0) { break; } - // Check if result[i].device[j].mac is all-zero - if(memcmp(result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", MAC_LENGTH) == 0) + // Check if mac is all-zero + if(memcmp(mac, "\x00\x00\x00\x00\x00\x00", MAC_LENGTH) == 0) { - // Copy MAC address to result[i].device[j].mac - memcpy(result[i].device[j].mac, sender_mac, MAC_LENGTH); + // Copy MAC address to mac + memcpy(mac, sender_mac, MAC_LENGTH); break; } } // Memorize that we have received a reply for this IP address - result[i].device[j].replied[scan_id]++; + thread_data->extreme ? + thread_data->result_extreme[i].device[j].replied[scan_id]++ : + thread_data->result[i].device[j].replied[scan_id]++; } // Read all ARP responses -static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, const char **error, - struct arp_result *result, const size_t result_len, const unsigned int scan_id) +static ssize_t read_arp(const int fd, struct thread_data *thread_data) { ssize_t ret = 0; unsigned char buffer[BUF_SIZE]; @@ -282,8 +296,8 @@ static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, } // Error - *error = strerror(errno); - printf("recvfrom(): %s", *error); + thread_data->error = strerror(errno); + printf("recvfrom(): %s", thread_data->error); break; } struct ethhdr *rcv_resp = (struct ethhdr *) buffer; @@ -310,7 +324,7 @@ static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, #ifdef DEBUG printf("%-16s %-20s\t%02x:%02x:%02x:%02x:%02x:%02x", - iface, inet_ntoa(sender_a), + thread_data->iface, inet_ntoa(sender_a), arp_resp->sender_mac[0], arp_resp->sender_mac[1], arp_resp->sender_mac[2], @@ -318,7 +332,7 @@ static ssize_t read_arp(const int fd, const char *iface, struct in_addr *dst_ip, arp_resp->sender_mac[4], arp_resp->sender_mac[5]); #endif - add_result(iface, &sender_a, dst_ip, arp_resp->sender_mac, result, result_len, scan_id); + add_result(&sender_a, arp_resp->sender_mac, thread_data, thread_data->num_scans); } return ret; @@ -401,19 +415,35 @@ static void *arp_scan_iface(void *args) // Allocate memory for ARP response buffer const size_t arp_result_len = 1 << (32 - thread_data->dst_cidr); - struct arp_result *result = calloc(arp_result_len, sizeof(struct arp_result)); - if(result == NULL) + thread_data->result_size = arp_result_len; + if(thread_data->extreme) { - // Memory allocation failed due to insufficient memory being - // available - thread_data->status = STATUS_ERROR; - thread_data->error = strerror(ENOMEM); - pthread_exit(NULL); + // Allocate extreme memory for ARP response buffer + struct arp_result_extreme *result = calloc(arp_result_len, sizeof(struct arp_result_extreme)); + if(result == NULL) + { + // Memory allocation failed due to insufficient memory being + // available + thread_data->status = STATUS_ERROR; + thread_data->error = strerror(ENOMEM); + pthread_exit(NULL); + } + thread_data->result_extreme = result; + } + else + { + // Allocate memory for ARP response buffer + struct arp_result *result = calloc(arp_result_len, sizeof(struct arp_result)); + if(result == NULL) + { + // Memory allocation failed due to insufficient memory being + // available + thread_data->status = STATUS_ERROR; + thread_data->error = strerror(ENOMEM); + pthread_exit(NULL); + } + thread_data->result = result; } - - // Store ARP response buffer in thread_data - thread_data->result_size = arp_result_len; - thread_data->result = result; for(thread_data->num_scans = 0; thread_data->num_scans < thread_data->total_scans; thread_data->num_scans++) { @@ -421,16 +451,14 @@ static void *arp_scan_iface(void *args) printf("Still scanning interface %s (%s/%i) %i%%...\n", iface, thread_data->ipstr, thread_data->dst_cidr, 100*scan_id/thread_data->total_scans); #endif // Send ARP requests to all IPs in subnet - if(send_arps(arp_socket, ifindex, iface, thread_data->mac, &thread_data->error, &thread_data->src_addr.sin_addr, - thread_data->dst_addr.sin_addr, thread_data->dst_cidr, &thread_data->scanned_addresses) != 0) + if(send_arps(arp_socket, ifindex, thread_data) != 0) { thread_data->status = STATUS_ERROR; break; } // Read ARP responses - if(read_arp(arp_socket, iface, &thread_data->dst_addr.sin_addr, &thread_data->error, - thread_data->result, thread_data->result_size, thread_data->num_scans) != 0) + if(read_arp(arp_socket, thread_data) != 0) { thread_data->status = STATUS_ERROR; break; @@ -466,14 +494,29 @@ static void print_results(struct thread_data *thread_data) } // Check if there are any results - unsigned int replies = 0; + bool any_replies = false; for(unsigned int i = 0; i < thread_data->result_size; i++) for(unsigned int j = 0; j < MAX_MACS; j++) for(unsigned int k = 0; k < thread_data->total_scans; k++) - replies += thread_data->result[i].device[j].replied[k]; + if(thread_data->extreme) + { + if(thread_data->result_extreme[i].device[j].replied[k]) + { + any_replies = true; + break; + } + } + else + { + if(thread_data->result[i].device[j].replied[k]) + { + any_replies = true; + break; + } + } // Exit early if there are no results - if(replies == 0) + if(!any_replies) { printf("No devices replied on interface %s (%s/%i)\n\n", thread_data->iface, thread_data->ipstr, thread_data->dst_cidr); @@ -489,7 +532,7 @@ static void print_results(struct thread_data *thread_data) // Add our own IP address to the results so IP conflicts can be detected // (our own IP address is not included in the ARP scan) for(unsigned int i = 0; i < thread_data->total_scans; i++) - add_result(thread_data->iface, &thread_data->src_addr.sin_addr, &thread_data->dst_addr.sin_addr, thread_data->mac, thread_data->result, thread_data->result_size, i); + add_result(&thread_data->src_addr.sin_addr, thread_data->mac, thread_data, i); // Print results for(unsigned int i = 0; i < thread_data->result_size; i++) @@ -500,18 +543,25 @@ static void print_results(struct thread_data *thread_data) for(j = 0; j < MAX_MACS; j++) { // Check if result[i].mac[j] is all-zero, if so, skip this entry - if(memcmp(thread_data->result[i].device[j].mac, "\x00\x00\x00\x00\x00\x00", 6) == 0) + unsigned char *mac = thread_data->extreme ? + thread_data->result_extreme[i].device[j].mac : + thread_data->result[i].device[j].mac; + if(memcmp(mac, "\x00\x00\x00\x00\x00\x00", 6) == 0) break; - // Check if IP address replied - replies = 0u; bool replied = false; - unsigned int multiple_replies = 0; + unsigned char replies = 0u; + unsigned char multiple_replies = 0; + const unsigned char *rp = thread_data->extreme ? + thread_data->result_extreme[i].device[j].replied : + thread_data->result[i].device[j].replied; + + // Check if IP address replied for(unsigned int k = 0; k < thread_data->total_scans; k++) { - replied |= thread_data->result[i].device[j].replied[k] > 0; - replies += thread_data->result[i].device[j].replied[k] > 0 ? 1 : 0; - multiple_replies += thread_data->result[i].device[j].replied[k] > 1; + replied |= rp[k] > 0; + replies += rp[k] > 0 ? 1 : 0; + multiple_replies += rp[k] > 1; } if(!replied) continue; @@ -528,28 +578,17 @@ static void print_results(struct thread_data *thread_data) printf("%-16s %-16s %-24s %02x:%02x:%02x:%02x:%02x:%02x %u %%\n", thread_data->ipstr, thread_data->iface, get_hostname(&ip), - thread_data->result[i].device[j].mac[0], - thread_data->result[i].device[j].mac[1], - thread_data->result[i].device[j].mac[2], - thread_data->result[i].device[j].mac[3], - thread_data->result[i].device[j].mac[4], - thread_data->result[i].device[j].mac[5], + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5], replies * 100 / thread_data->total_scans); #ifdef DEBUG for(unsigned int k = 0; k < thread_data->total_scans; k++) - printf(" %s", thread_data->result[i].device[j].replied[k] > 0 ? "X" : "-"); + printf(" %s", rp[k] > 0 ? "X" : "-"); #endif if(multiple_replies > 0) - printf("WARNING: Received multiple replies from %02x:%02x:%02x:%02x:%02x:%02x for %s in %i scan%s\n", - thread_data->result[i].device[j].mac[0], - thread_data->result[i].device[j].mac[1], - thread_data->result[i].device[j].mac[2], - thread_data->result[i].device[j].mac[3], - thread_data->result[i].device[j].mac[4], - thread_data->result[i].device[j].mac[5], - thread_data->ipstr, - multiple_replies, multiple_replies > 1 ? "s" : ""); + printf("INFO: Received multiple replies from %02x:%02x:%02x:%02x:%02x:%02x for %s in %i scan%s\n", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5], + thread_data->ipstr, multiple_replies, multiple_replies > 1 ? "s" : ""); } // Print warning if we received multiple replies @@ -600,6 +639,7 @@ int run_arp_scan(const bool scan_all, const bool extreme_mode) memcpy(&thread_data[tid].src_addr, tmp->ifa_addr, sizeof(thread_data[tid].src_addr)); inet_ntop(AF_INET, &thread_data[tid].src_addr.sin_addr, thread_data[tid].ipstr, INET_ADDRSTRLEN); + thread_data[tid].extreme = extreme_mode; thread_data[tid].scan_all = scan_all || extreme_mode; thread_data[tid].total_scans = extreme_mode ? 10*NUM_SCANS : NUM_SCANS; From b66f7f1ba3dacbded5b5f21f3a53262acc71ab1f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 May 2023 10:13:42 +0200 Subject: [PATCH 58/78] Update SQLite3 to 3.42.0 Signed-off-by: DL6ER --- src/database/shell.c | 1029 ++++-- src/database/sqlite3.c | 7045 +++++++++++++++++++++++++++------------- src/database/sqlite3.h | 202 +- 3 files changed, 5574 insertions(+), 2702 deletions(-) diff --git a/src/database/shell.c b/src/database/shell.c index ea591be3f..647a21422 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -117,6 +117,7 @@ typedef unsigned short int u16; #include #include #include +#include #include "sqlite3.h" typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; @@ -126,8 +127,6 @@ typedef unsigned char u8; #endif #include #include -// print_FTL_version() -#include "../log.h" #if !defined(_WIN32) && !defined(WIN32) # include @@ -247,6 +246,7 @@ typedef unsigned char u8; #if SQLITE_OS_WINRT #include #endif +#define WIN32_LEAN_AND_MEAN #include /* string conversion routines only needed on Win32 */ @@ -461,11 +461,25 @@ static void endTimer(void){ static int bail_on_error = 0; /* -** Threat stdin as an interactive input if the following variable +** Treat stdin as an interactive input if the following variable ** is true. Otherwise, assume stdin is connected to a file or pipe. */ static int stdin_is_interactive = 1; +#if (defined(_WIN32) || defined(WIN32)) && SHELL_USE_LOCAL_GETLINE \ + && !defined(SHELL_OMIT_WIN_UTF8) +# define SHELL_WIN_UTF8_OPT 1 +#else +# define SHELL_WIN_UTF8_OPT 0 +#endif + +#if SHELL_WIN_UTF8_OPT +/* +** Setup console for UTF-8 input/output when following variable true. +*/ +static int console_utf8 = 0; +#endif + /* ** On Windows systems we have to know if standard output is a console ** in order to translate UTF-8 into MBCS. The following variable is @@ -598,16 +612,150 @@ static char *dynamicContinuePrompt(void){ } #endif /* !defined(SQLITE_OMIT_DYNAPROMPT) */ +#if SHELL_WIN_UTF8_OPT +/* Following struct is used for -utf8 operation. */ +static struct ConsoleState { + int stdinEof; /* EOF has been seen on console input */ + int infsMode; /* Input file stream mode upon shell start */ + UINT inCodePage; /* Input code page upon shell start */ + UINT outCodePage; /* Output code page upon shell start */ + HANDLE hConsoleIn; /* Console input handle */ + DWORD consoleMode; /* Console mode upon shell start */ +} conState = { 0, 0, 0, 0, INVALID_HANDLE_VALUE, 0 }; + +#ifndef _O_U16TEXT /* For build environments lacking this constant: */ +# define _O_U16TEXT 0x20000 +#endif + +/* +** Prepare console, (if known to be a WIN32 console), for UTF-8 +** input (from either typing or suitable paste operations) and for +** UTF-8 rendering. This may "fail" with a message to stderr, where +** the preparation is not done and common "code page" issues occur. +*/ +static void console_prepare(void){ + HANDLE hCI = GetStdHandle(STD_INPUT_HANDLE); + DWORD consoleMode = 0; + if( isatty(0) && GetFileType(hCI)==FILE_TYPE_CHAR + && GetConsoleMode( hCI, &consoleMode) ){ + if( !IsValidCodePage(CP_UTF8) ){ + fprintf(stderr, "Cannot use UTF-8 code page.\n"); + console_utf8 = 0; + return; + } + conState.hConsoleIn = hCI; + conState.consoleMode = consoleMode; + conState.inCodePage = GetConsoleCP(); + conState.outCodePage = GetConsoleOutputCP(); + SetConsoleCP(CP_UTF8); + SetConsoleOutputCP(CP_UTF8); + consoleMode |= ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT; + SetConsoleMode(conState.hConsoleIn, consoleMode); + conState.infsMode = _setmode(_fileno(stdin), _O_U16TEXT); + console_utf8 = 1; + }else{ + console_utf8 = 0; + } +} + +/* +** Undo the effects of console_prepare(), if any. +*/ +static void SQLITE_CDECL console_restore(void){ + if( console_utf8 && conState.inCodePage!=0 + && conState.hConsoleIn!=INVALID_HANDLE_VALUE ){ + _setmode(_fileno(stdin), conState.infsMode); + SetConsoleCP(conState.inCodePage); + SetConsoleOutputCP(conState.outCodePage); + SetConsoleMode(conState.hConsoleIn, conState.consoleMode); + /* Avoid multiple calls. */ + conState.hConsoleIn = INVALID_HANDLE_VALUE; + conState.consoleMode = 0; + console_utf8 = 0; + } +} + +/* +** Collect input like fgets(...) with special provisions for input +** from the Windows console to get around its strange coding issues. +** Defers to plain fgets() when input is not interactive or when the +** startup option, -utf8, has not been provided or taken effect. +*/ +static char* utf8_fgets(char *buf, int ncmax, FILE *fin){ + if( fin==0 ) fin = stdin; + if( fin==stdin && stdin_is_interactive && console_utf8 ){ +# define SQLITE_IALIM 150 + wchar_t wbuf[SQLITE_IALIM]; + int lend = 0; + int noc = 0; + if( ncmax==0 || conState.stdinEof ) return 0; + buf[0] = 0; + while( noc SQLITE_IALIM*4+1 + noc) + ? SQLITE_IALIM : (ncmax-1 - noc)/4; +# undef SQLITE_IALIM + DWORD nbr = 0; + BOOL bRC = ReadConsoleW(conState.hConsoleIn, wbuf, na, &nbr, 0); + if( !bRC || (noc==0 && nbr==0) ) return 0; + if( nbr > 0 ){ + int nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, + wbuf,nbr,0,0,0,0); + if( nmb !=0 && noc+nmb <= ncmax ){ + int iseg = noc; + nmb = WideCharToMultiByte(CP_UTF8,WC_COMPOSITECHECK|WC_DEFAULTCHAR, + wbuf,nbr,buf+noc,nmb,0,0); + noc += nmb; + /* Fixup line-ends as coded by Windows for CR (or "Enter".)*/ + if( noc > 0 ){ + if( buf[noc-1]=='\n' ){ + lend = 1; + if( noc > 1 && buf[noc-2]=='\r' ){ + buf[noc-2] = '\n'; + --noc; + } + } + } + /* Check for ^Z (anywhere in line) too. */ + while( iseg < noc ){ + if( buf[iseg]==0x1a ){ + conState.stdinEof = 1; + noc = iseg; /* Chop ^Z and anything following. */ + break; + } + ++iseg; + } + }else break; /* Drop apparent garbage in. (Could assert.) */ + }else break; + } + /* If got nothing, (after ^Z chop), must be at end-of-file. */ + if( noc == 0 ) return 0; + buf[noc] = 0; + return buf; + }else{ + return fgets(buf, ncmax, fin); + } +} + +# define fgets(b,n,f) utf8_fgets(b,n,f) +#endif /* SHELL_WIN_UTF8_OPT */ + /* ** Render output like fprintf(). Except, if the output is going to the -** console and if this is running on a Windows machine, translate the -** output from UTF-8 into MBCS. +** console and if this is running on a Windows machine, and if the -utf8 +** option is unavailable or (available and inactive), translate the +** output from UTF-8 into MBCS for output through 8-bit stdout stream. +** (With -utf8 active, no translation is needed and must not be done.) */ #if defined(_WIN32) || defined(WIN32) void utf8_printf(FILE *out, const char *zFormat, ...){ va_list ap; va_start(ap, zFormat); - if( stdout_is_console && (out==stdout || out==stderr) ){ + if( stdout_is_console && (out==stdout || out==stderr) +# if SHELL_WIN_UTF8_OPT + && !console_utf8 +# endif + ){ char *z1 = sqlite3_vmprintf(zFormat, ap); char *z2 = sqlite3_win32_utf8_to_mbcs_v2(z1, 0); sqlite3_free(z1); @@ -639,7 +787,7 @@ static void shell_out_of_memory(void){ /* Check a pointer to see if it is NULL. If it is NULL, exit with an ** out-of-memory error. */ -static void shell_check_oom(void *p){ +static void shell_check_oom(const void *p){ if( p==0 ) shell_out_of_memory(); } @@ -818,9 +966,14 @@ static char *local_getline(char *zLine, FILE *in){ } } #if defined(_WIN32) || defined(WIN32) - /* For interactive input on Windows systems, translate the - ** multi-byte characterset characters into UTF-8. */ - if( stdin_is_interactive && in==stdin ){ + /* For interactive input on Windows systems, without -utf8, + ** translate the multi-byte characterset characters into UTF-8. + ** This is the translation that predates the -utf8 option. */ + if( stdin_is_interactive && in==stdin +# if SHELL_WIN_UTF8_OPT + && !console_utf8 +# endif /* SHELL_WIN_UTF8_OPT */ + ){ char *zTrans = sqlite3_win32_mbcs_to_utf8_v2(zLine, 0); if( zTrans ){ i64 nTrans = strlen(zTrans)+1; @@ -861,10 +1014,21 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ #if SHELL_USE_LOCAL_GETLINE printf("%s", zPrompt); fflush(stdout); - zResult = local_getline(zPrior, stdin); + do{ + zResult = local_getline(zPrior, stdin); + zPrior = 0; + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + if( zResult==0 ) sqlite3_sleep(50); + }while( zResult==0 && seenInterrupt>0 ); #else free(zPrior); zResult = shell_readline(zPrompt); + while( zResult==0 ){ + /* ^C trap creates a false EOF, so let "interrupt" thread catch up. */ + sqlite3_sleep(50); + if( seenInterrupt==0 ) break; + zResult = shell_readline(""); + } if( zResult && *zResult ) shell_add_history(zResult); #endif } @@ -1004,6 +1168,7 @@ static void appendText(ShellText *p, const char *zAppend, char quote){ */ static char quoteChar(const char *zName){ int i; + if( zName==0 ) return '"'; if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; for(i=0; zName[i]; i++){ if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; @@ -1647,7 +1812,8 @@ int sqlite3MemTraceDeactivate(void){ ** ****************************************************************************** ** -** This SQLite extension implements functions that compute SHA3 hashes. +** This SQLite extension implements functions that compute SHA3 hashes +** in the way described by the (U.S.) NIST FIPS 202 SHA-3 Standard. ** Two SQL functions are implemented: ** ** sha3(X,SIZE) @@ -3175,6 +3341,7 @@ SQLITE_EXTENSION_INIT1; #define U8_TYPEDEF #endif +/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ static const u8 b64DigitValues[128] = { /* HT LF VT FF CR */ ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, @@ -3246,9 +3413,9 @@ static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ } /* Skip over text which is not base64 numeral(s). */ -static char * skipNonB64( char *s ){ +static char * skipNonB64( char *s, int nc ){ char c; - while( (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; return s; } @@ -3257,7 +3424,7 @@ static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 && *pIn!=PAD_CHAR ){ static signed char nboi[] = { 0, 0, 1, 2, 3 }; - char *pUse = skipNonB64(pIn); + char *pUse = skipNonB64(pIn, ncIn); unsigned long qv = 0L; int nti, nbo, nac; ncIn -= (pUse - pIn); @@ -3317,9 +3484,16 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "blob expanded to base64 too big", -1); return; } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } cBuf = sqlite3_malloc(nc); if( !cBuf ) goto memFail; - bBuf = (u8*)sqlite3_value_blob(av[0]); nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); break; @@ -3332,9 +3506,16 @@ static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ }else if( nb<1 ){ nb = 1; } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } bBuf = sqlite3_malloc(nb); if( !bBuf ) goto memFail; - cBuf = (char *)sqlite3_value_text(av[0]); nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); break; @@ -3522,9 +3703,9 @@ static u8 base85DigitValue( char c ){ #define B85_DARK_MAX 80 -static char * skipNonB85( char *s ){ +static char * skipNonB85( char *s, int nc ){ char c; - while( (c = *s) && !IS_B85(c) ) ++s; + while( nc-- > 0 && (c = *s) && !IS_B85(c) ) ++s; return s; } @@ -3594,7 +3775,7 @@ static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; while( ncIn>0 ){ static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; - char *pUse = skipNonB85(pIn); + char *pUse = skipNonB85(pIn, ncIn); unsigned long qv = 0L; int nti, nbo; ncIn -= (pUse - pIn); @@ -3679,9 +3860,16 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ sqlite3_result_error(context, "blob expanded to base85 too big", -1); return; } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } cBuf = sqlite3_malloc(nc); if( !cBuf ) goto memFail; - bBuf = (u8*)sqlite3_value_blob(av[0]); nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); sqlite3_result_text(context, cBuf, nc, sqlite3_free); break; @@ -3694,9 +3882,16 @@ static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ }else if( nb<1 ){ nb = 1; } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } bBuf = sqlite3_malloc(nb); if( !bBuf ) goto memFail; - cBuf = (char *)sqlite3_value_text(av[0]); nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); sqlite3_result_blob(context, bBuf, nb, sqlite3_free); break; @@ -4116,7 +4311,7 @@ int sqlite3_ieee_init( /************************* End ../ext/misc/ieee754.c ********************/ /************************* Begin ../ext/misc/series.c ******************/ /* -** 2015-08-18 +** 2015-08-18, 2023-04-28 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -4129,7 +4324,19 @@ int sqlite3_ieee_init( ** ** This file demonstrates how to create a table-valued-function using ** a virtual table. This demo implements the generate_series() function -** which gives similar results to the eponymous function in PostgreSQL. +** which gives the same results as the eponymous function in PostgreSQL, +** within the limitation that its arguments are signed 64-bit integers. +** +** Considering its equivalents to generate_series(start,stop,step): A +** value V[n] sequence is produced for integer n ascending from 0 where +** ( V[n] == start + n * step && sgn(V[n] - stop) * sgn(step) >= 0 ) +** for each produced value (independent of production time ordering.) +** +** All parameters must be either integer or convertable to integer. +** The start parameter is required. +** The stop parameter defaults to (1<<32)-1 (aka 4294967295 or 0xffffffff) +** The step parameter defaults to 1 and 0 is treated as 1. +** ** Examples: ** ** SELECT * FROM generate_series(0,100,5); @@ -4145,6 +4352,14 @@ int sqlite3_ieee_init( ** ** Integers 20 through 29. ** +** SELECT * FROM generate_series(0,-100,-5); +** +** Integers 0 -5 -10 ... -100. +** +** SELECT * FROM generate_series(0,-1); +** +** Empty sequence. +** ** HOW IT WORKS ** ** The generate_series "function" is really a virtual table with the @@ -4157,6 +4372,9 @@ int sqlite3_ieee_init( ** step HIDDEN ** ); ** +** The virtual table also has a rowid, logically equivalent to n+1 where +** "n" is the ascending integer in the aforesaid production definition. +** ** Function arguments in queries against this virtual table are translated ** into equality constraints against successive hidden columns. In other ** words, the following pairs of queries are equivalent to each other: @@ -4189,9 +4407,126 @@ int sqlite3_ieee_init( SQLITE_EXTENSION_INIT1 #include #include +#include #ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return that member of a generate_series(...) sequence whose 0-based +** index is ix. The 0th member is given by smBase. The sequence members +** progress per ix increment by smStep. +*/ +static sqlite3_int64 genSeqMember(sqlite3_int64 smBase, + sqlite3_int64 smStep, + sqlite3_uint64 ix){ + if( ix>=(sqlite3_uint64)LLONG_MAX ){ + /* Get ix into signed i64 range. */ + ix -= (sqlite3_uint64)LLONG_MAX; + /* With 2's complement ALU, this next can be 1 step, but is split into + * 2 for UBSAN's satisfaction (and hypothetical 1's complement ALUs.) */ + smBase += (LLONG_MAX/2) * smStep; + smBase += (LLONG_MAX - LLONG_MAX/2) * smStep; + } + /* Under UBSAN (or on 1's complement machines), must do this last term + * in steps to avoid the dreaded (and harmless) signed multiply overlow. */ + if( ix>=2 ){ + sqlite3_int64 ix2 = (sqlite3_int64)ix/2; + smBase += ix2*smStep; + ix -= ix2; + } + return smBase + ((sqlite3_int64)ix)*smStep; +} + +/* typedef unsigned char u8; */ + +typedef struct SequenceSpec { + sqlite3_int64 iBase; /* Starting value ("start") */ + sqlite3_int64 iTerm; /* Given terminal value ("stop") */ + sqlite3_int64 iStep; /* Increment ("step") */ + sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */ + sqlite3_uint64 uSeqIndexNow; /* Current index during generation */ + sqlite3_int64 iValueNow; /* Current value during generation */ + u8 isNotEOF; /* Sequence generation not exhausted */ + u8 isReversing; /* Sequence is being reverse generated */ +} SequenceSpec; + +/* +** Prepare a SequenceSpec for use in generating an integer series +** given initialized iBase, iTerm and iStep values. Sequence is +** initialized per given isReversing. Other members are computed. +*/ +static void setupSequence( SequenceSpec *pss ){ + int bSameSigns; + pss->uSeqIndexMax = 0; + pss->isNotEOF = 0; + bSameSigns = (pss->iBase < 0)==(pss->iTerm < 0); + if( pss->iTerm < pss->iBase ){ + sqlite3_uint64 nuspan = 0; + if( bSameSigns ){ + nuspan = (sqlite3_uint64)(pss->iBase - pss->iTerm); + }else{ + /* Under UBSAN (or on 1's complement machines), must do this in steps. + * In this clause, iBase>=0 and iTerm<0 . */ + nuspan = 1; + nuspan += pss->iBase; + nuspan += -(pss->iTerm+1); + } + if( pss->iStep<0 ){ + pss->isNotEOF = 1; + if( nuspan==ULONG_MAX ){ + pss->uSeqIndexMax = ( pss->iStep>LLONG_MIN )? nuspan/-pss->iStep : 1; + }else if( pss->iStep>LLONG_MIN ){ + pss->uSeqIndexMax = nuspan/-pss->iStep; + } + } + }else if( pss->iTerm > pss->iBase ){ + sqlite3_uint64 puspan = 0; + if( bSameSigns ){ + puspan = (sqlite3_uint64)(pss->iTerm - pss->iBase); + }else{ + /* Under UBSAN (or on 1's complement machines), must do this in steps. + * In this clause, iTerm>=0 and iBase<0 . */ + puspan = 1; + puspan += pss->iTerm; + puspan += -(pss->iBase+1); + } + if( pss->iStep>0 ){ + pss->isNotEOF = 1; + pss->uSeqIndexMax = puspan/pss->iStep; + } + }else if( pss->iTerm == pss->iBase ){ + pss->isNotEOF = 1; + pss->uSeqIndexMax = 0; + } + pss->uSeqIndexNow = (pss->isReversing)? pss->uSeqIndexMax : 0; + pss->iValueNow = (pss->isReversing) + ? genSeqMember(pss->iBase, pss->iStep, pss->uSeqIndexMax) + : pss->iBase; +} +/* +** Progress sequence generator to yield next value, if any. +** Leave its state to either yield next value or be at EOF. +** Return whether there is a next value, or 0 at EOF. +*/ +static int progressSequence( SequenceSpec *pss ){ + if( !pss->isNotEOF ) return 0; + if( pss->isReversing ){ + if( pss->uSeqIndexNow > 0 ){ + pss->uSeqIndexNow--; + pss->iValueNow -= pss->iStep; + }else{ + pss->isNotEOF = 0; + } + }else{ + if( pss->uSeqIndexNow < pss->uSeqIndexMax ){ + pss->uSeqIndexNow++; + pss->iValueNow += pss->iStep; + }else{ + pss->isNotEOF = 0; + } + } + return pss->isNotEOF; +} /* series_cursor is a subclass of sqlite3_vtab_cursor which will ** serve as the underlying representation of a cursor that scans @@ -4200,12 +4535,7 @@ SQLITE_EXTENSION_INIT1 typedef struct series_cursor series_cursor; struct series_cursor { sqlite3_vtab_cursor base; /* Base class - must be first */ - int isDesc; /* True to count down rather than up */ - sqlite3_int64 iRowid; /* The rowid */ - sqlite3_int64 iValue; /* Current value ("value") */ - sqlite3_int64 mnValue; /* Mimimum value ("start") */ - sqlite3_int64 mxValue; /* Maximum value ("stop") */ - sqlite3_int64 iStep; /* Increment ("step") */ + SequenceSpec ss; /* (this) Derived class data */ }; /* @@ -4287,12 +4617,7 @@ static int seriesClose(sqlite3_vtab_cursor *cur){ */ static int seriesNext(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - if( pCur->isDesc ){ - pCur->iValue -= pCur->iStep; - }else{ - pCur->iValue += pCur->iStep; - } - pCur->iRowid++; + progressSequence( & pCur->ss ); return SQLITE_OK; } @@ -4308,23 +4633,23 @@ static int seriesColumn( series_cursor *pCur = (series_cursor*)cur; sqlite3_int64 x = 0; switch( i ){ - case SERIES_COLUMN_START: x = pCur->mnValue; break; - case SERIES_COLUMN_STOP: x = pCur->mxValue; break; - case SERIES_COLUMN_STEP: x = pCur->iStep; break; - default: x = pCur->iValue; break; + case SERIES_COLUMN_START: x = pCur->ss.iBase; break; + case SERIES_COLUMN_STOP: x = pCur->ss.iTerm; break; + case SERIES_COLUMN_STEP: x = pCur->ss.iStep; break; + default: x = pCur->ss.iValueNow; break; } sqlite3_result_int64(ctx, x); return SQLITE_OK; } /* -** Return the rowid for the current row. In this implementation, the -** first row returned is assigned rowid value 1, and each subsequent -** row a value 1 more than that of the previous. +** Return the rowid for the current row, logically equivalent to n+1 where +** "n" is the ascending integer in the aforesaid production definition. */ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ series_cursor *pCur = (series_cursor*)cur; - *pRowid = pCur->iRowid; + sqlite3_uint64 n = pCur->ss.uSeqIndexNow; + *pRowid = (sqlite3_int64)((n<0xffffffffffffffff)? n+1 : 0); return SQLITE_OK; } @@ -4334,14 +4659,10 @@ static int seriesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ */ static int seriesEof(sqlite3_vtab_cursor *cur){ series_cursor *pCur = (series_cursor*)cur; - if( pCur->isDesc ){ - return pCur->iValue < pCur->mnValue; - }else{ - return pCur->iValue > pCur->mxValue; - } + return !pCur->ss.isNotEOF; } -/* True to cause run-time checking of the start=, stop=, and/or step= +/* True to cause run-time checking of the start=, stop=, and/or step= ** parameters. The only reason to do this is for testing the ** constraint checking logic for virtual tables in the SQLite core. */ @@ -4352,7 +4673,7 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ /* ** This method is called to "rewind" the series_cursor object back ** to the first row of output. This method is always called at least -** once prior to any call to seriesColumn() or seriesRowid() or +** once prior to any call to seriesColumn() or seriesRowid() or ** seriesEof(). ** ** The query plan selected by seriesBestIndex is passed in the idxNum @@ -4372,7 +4693,7 @@ static int seriesEof(sqlite3_vtab_cursor *cur){ ** (so that seriesEof() will return true) if the table is empty. */ static int seriesFilter( - sqlite3_vtab_cursor *pVtabCursor, + sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStrUnused, int argc, sqlite3_value **argv ){ @@ -4380,46 +4701,41 @@ static int seriesFilter( int i = 0; (void)idxStrUnused; if( idxNum & 1 ){ - pCur->mnValue = sqlite3_value_int64(argv[i++]); + pCur->ss.iBase = sqlite3_value_int64(argv[i++]); }else{ - pCur->mnValue = 0; + pCur->ss.iBase = 0; } if( idxNum & 2 ){ - pCur->mxValue = sqlite3_value_int64(argv[i++]); + pCur->ss.iTerm = sqlite3_value_int64(argv[i++]); }else{ - pCur->mxValue = 0xffffffff; + pCur->ss.iTerm = 0xffffffff; } if( idxNum & 4 ){ - pCur->iStep = sqlite3_value_int64(argv[i++]); - if( pCur->iStep==0 ){ - pCur->iStep = 1; - }else if( pCur->iStep<0 ){ - pCur->iStep = -pCur->iStep; + pCur->ss.iStep = sqlite3_value_int64(argv[i++]); + if( pCur->ss.iStep==0 ){ + pCur->ss.iStep = 1; + }else if( pCur->ss.iStep<0 ){ if( (idxNum & 16)==0 ) idxNum |= 8; } }else{ - pCur->iStep = 1; + pCur->ss.iStep = 1; } for(i=0; imnValue = 1; - pCur->mxValue = 0; + pCur->ss.iBase = 1; + pCur->ss.iTerm = 0; + pCur->ss.iStep = 1; break; } } if( idxNum & 8 ){ - pCur->isDesc = 1; - pCur->iValue = pCur->mxValue; - if( pCur->iStep>0 ){ - pCur->iValue -= (pCur->mxValue - pCur->mnValue)%pCur->iStep; - } + pCur->ss.isReversing = pCur->ss.iStep > 0; }else{ - pCur->isDesc = 0; - pCur->iValue = pCur->mnValue; + pCur->ss.isReversing = pCur->ss.iStep < 0; } - pCur->iRowid = 1; + setupSequence( &pCur->ss ); return SQLITE_OK; } @@ -5174,7 +5490,7 @@ static const char *re_subcompile_string(ReCompiled *p){ break; } case '[': { - int iFirst = p->nState; + unsigned int iFirst = p->nState; if( rePeek(p)=='^' ){ re_append(p, RE_OP_CC_EXC, 0); p->sIn.i++; @@ -5198,7 +5514,7 @@ static const char *re_subcompile_string(ReCompiled *p){ if( rePeek(p)==']' ){ p->sIn.i++; break; } } if( c==0 ) return "unclosed '['"; - p->aArg[iFirst] = p->nState - iFirst; + if( p->nState>iFirst ) p->aArg[iFirst] = p->nState - iFirst; break; } case '\\': { @@ -8775,7 +9091,10 @@ static int zipfileColumn( ** it to be a directory either if the mode suggests so, or if ** the final character in the name is '/'. */ u32 mode = pCDS->iExternalAttr >> 16; - if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){ + if( !(mode & S_IFDIR) + && pCDS->nFile>=1 + && pCDS->zFile[pCDS->nFile-1]!='/' + ){ sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC); } } @@ -9212,9 +9531,19 @@ static u32 zipfileGetTime(sqlite3_value *pVal){ */ static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){ if( pOld ){ - ZipfileEntry **pp; - for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext)); - *pp = (*pp)->pNext; + if( pTab->pFirstEntry==pOld ){ + pTab->pFirstEntry = pOld->pNext; + if( pTab->pLastEntry==pOld ) pTab->pLastEntry = 0; + }else{ + ZipfileEntry *p; + for(p=pTab->pFirstEntry; p; p=p->pNext){ + if( p->pNext==pOld ){ + p->pNext = pOld->pNext; + if( pTab->pLastEntry==pOld ) pTab->pLastEntry = p; + break; + } + } + } zipfileEntryFree(pOld); } } @@ -12680,6 +13009,7 @@ static int dbdataConnect( (void)argc; (void)argv; (void)pzErr; + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); if( rc==SQLITE_OK ){ pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); if( pTab==0 ){ @@ -13025,10 +13355,14 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); if( rc!=SQLITE_OK ) return rc; - if( pCsr->aPage ) break; + if( pCsr->aPage && pCsr->nPage>=256 ) break; + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; } + + assert( iOff+3+2<=pCsr->nPage ); pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); } @@ -13263,8 +13597,7 @@ static int dbdataGetEncoding(DbdataCursor *pCsr){ int nPg1 = 0; u8 *aPg1 = 0; rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); - assert( rc!=SQLITE_OK || nPg1==0 || nPg1>=512 ); - if( rc==SQLITE_OK && nPg1>0 ){ + if( rc==SQLITE_OK && nPg1>=(56+4) ){ pCsr->enc = get_uint32(&aPg1[56]); } sqlite3_free(aPg1); @@ -13322,8 +13655,6 @@ static int dbdataFilter( } if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); - }else{ - pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } /* Try to determine the encoding of the db by inspecting the header @@ -13332,6 +13663,10 @@ static int dbdataFilter( rc = dbdataGetEncoding(pCsr); } + if( rc!=SQLITE_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + } + if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); } @@ -14570,7 +14905,7 @@ static void recoverAddTable( int iField = sqlite3_column_int(pStmt, 0); int iCol = sqlite3_column_int(pStmt, 1); - assert( iFieldnCol && iColnCol ); + assert( iColnCol ); pNew->aCol[iCol].iField = iField; pNew->bIntkey = 0; @@ -16334,7 +16669,7 @@ int sqlite3_recover_finish(sqlite3_recover *p){ #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ /************************* End ../ext/recover/sqlite3recover.c ********************/ -# endif +# endif /* SQLITE_HAVE_SQLITE3R */ #endif #ifdef SQLITE_SHELL_EXTSRC # include SHELL_STRINGIFY(SQLITE_SHELL_EXTSRC) @@ -16516,6 +16851,7 @@ static ShellState shellState; #define SHFLG_HeaderSet 0x00000080 /* showHeader has been specified */ #define SHFLG_DumpDataOnly 0x00000100 /* .dump show data only */ #define SHFLG_DumpNoSys 0x00000200 /* .dump omits system tables */ +#define SHFLG_TestingMode 0x00000400 /* allow unsafe testing features */ /* ** Macros for testing and setting shellFlgs @@ -16759,6 +17095,7 @@ static void editFunc( }else{ /* If the file did not originally contain \r\n then convert any new ** \r\n back into \n */ + p[sz] = 0; for(i=j=0; i2 ) exit(1); + if( ++seenInterrupt>1 ) exit(1); if( globalDb ) sqlite3_interrupt(globalDb); } @@ -17237,6 +17575,7 @@ static void printSchemaLine(FILE *out, const char *z, const char *zTail){ int i; for(i=0; iout, "1e999"); + raw_printf(p->out, "9.0e+999"); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-1e999"); + raw_printf(p->out, "-9.0e+999"); }else{ sqlite3_int64 ir = (sqlite3_int64)r; if( r==(double)ir ){ @@ -17717,9 +18056,9 @@ static int shell_callback( sqlite3_uint64 ur; memcpy(&ur,&r,sizeof(r)); if( ur==0x7ff0000000000000LL ){ - raw_printf(p->out, "1e999"); + raw_printf(p->out, "9.0e+999"); }else if( ur==0xfff0000000000000LL ){ - raw_printf(p->out, "-1e999"); + raw_printf(p->out, "-9.0e+999"); }else{ sqlite3_snprintf(50,z,"%!.20g", r); raw_printf(p->out, "%s", z); @@ -17923,6 +18262,7 @@ static char *shell_error_context(const char *zSql, sqlite3 *db){ if( db==0 || zSql==0 || (iOffset = sqlite3_error_offset(db))<0 + || iOffset>=(int)strlen(zSql) ){ return sqlite3_mprintf(""); } @@ -17934,7 +18274,7 @@ static char *shell_error_context(const char *zSql, sqlite3 *db){ len = strlen(zSql); if( len>78 ){ len = 78; - while( (zSql[len]&0xc0)==0x80 ) len--; + while( len>0 && (zSql[len]&0xc0)==0x80 ) len--; } zCode = sqlite3_mprintf("%.*s", len, zSql); shell_check_oom(zCode); @@ -18535,12 +18875,13 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ if( nVar==0 ) return; /* Nothing to do */ if( sqlite3_table_column_metadata(pArg->db, "TEMP", "sqlite_parameters", "key", 0, 0, 0, 0, 0)!=SQLITE_OK ){ - return; /* Parameter table does not exist */ + rc = SQLITE_NOTFOUND; + pQ = 0; + }else{ + rc = sqlite3_prepare_v2(pArg->db, + "SELECT value FROM temp.sqlite_parameters" + " WHERE key=?1", -1, &pQ, 0); } - rc = sqlite3_prepare_v2(pArg->db, - "SELECT value FROM temp.sqlite_parameters" - " WHERE key=?1", -1, &pQ, 0); - if( rc || pQ==0 ) return; for(i=1; i<=nVar; i++){ char zNum[30]; const char *zVar = sqlite3_bind_parameter_name(pStmt, i); @@ -18549,8 +18890,16 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){ zVar = zNum; } sqlite3_bind_text(pQ, 1, zVar, -1, SQLITE_STATIC); - if( sqlite3_step(pQ)==SQLITE_ROW ){ + if( rc==SQLITE_OK && pQ && sqlite3_step(pQ)==SQLITE_ROW ){ sqlite3_bind_value(pStmt, i, sqlite3_column_value(pQ, 0)); +#ifdef NAN + }else if( sqlite3_strlike("_NAN", zVar, 0)==0 ){ + sqlite3_bind_double(pStmt, i, NAN); +#endif +#ifdef INFINITY + }else if( sqlite3_strlike("_INF", zVar, 0)==0 ){ + sqlite3_bind_double(pStmt, i, INFINITY); +#endif }else{ sqlite3_bind_null(pStmt, i); } @@ -18794,7 +19143,7 @@ static void exec_prepared_stmt_columnar( azData = sqlite3_malloc64( nAlloc*sizeof(char*) ); shell_check_oom(azData); azNextLine = sqlite3_malloc64( nColumn*sizeof(char*) ); - shell_check_oom((void*)azNextLine); + shell_check_oom(azNextLine); memset((void*)azNextLine, 0, nColumn*sizeof(char*) ); if( p->cmOpts.bQuote ){ azQuoted = sqlite3_malloc64( nColumn*sizeof(char*) ); @@ -18824,6 +19173,7 @@ static void exec_prepared_stmt_columnar( } if( wx<0 ) wx = -wx; uz = (const unsigned char*)sqlite3_column_name(pStmt,i); + if( uz==0 ) uz = (u8*)""; azData[i] = translateForDisplayAndDup(uz, &zNotUsed, wx, bw); } do{ @@ -19757,13 +20107,13 @@ static const char *(azHelp[]) = { " input text.", #endif #ifndef SQLITE_OMIT_TEST_CONTROL - ".imposter INDEX TABLE Create imposter table TABLE on index INDEX", + ",imposter INDEX TABLE Create imposter table TABLE on index INDEX", #endif ".indexes ?TABLE? Show names of indexes", " If TABLE is specified, only show indexes for", " tables matching TABLE using the LIKE operator.", #ifdef SQLITE_ENABLE_IOTRACE - ".iotrace FILE Enable I/O diagnostic logging to FILE", + ",iotrace FILE Enable I/O diagnostic logging to FILE", #endif ".limit ?LIMIT? ?VAL? Display or change the value of an SQLITE_LIMIT", ".lint OPTIONS Report potential schema issues.", @@ -19772,8 +20122,10 @@ static const char *(azHelp[]) = { #if !defined(SQLITE_OMIT_LOAD_EXTENSION) && !defined(SQLITE_SHELL_FIDDLE) ".load FILE ?ENTRY? Load an extension library", #endif -#ifndef SQLITE_SHELL_FIDDLE - ".log FILE|off Turn logging on or off. FILE can be stderr/stdout", +#if !defined(SQLITE_SHELL_FIDDLE) + ".log FILE|on|off Turn logging on or off. FILE can be stderr/stdout", +#else + ".log on|off Turn logging on or off.", #endif ".mode MODE ?OPTIONS? Set output mode", " MODE is one of:", @@ -19870,7 +20222,7 @@ static const char *(azHelp[]) = { " Options:", " --indent Try to pretty-print the schema", " --nosys Omit objects whose names start with \"sqlite_\"", - ".selftest ?OPTIONS? Run tests defined in the SELFTEST table", + ",selftest ?OPTIONS? Run tests defined in the SELFTEST table", " Options:", " --init Create a new SELFTEST table", " -v Verbose output", @@ -19912,9 +20264,9 @@ static const char *(azHelp[]) = { #endif ".tables ?TABLE? List names of tables matching LIKE pattern TABLE", #ifndef SQLITE_SHELL_FIDDLE - ".testcase NAME Begin redirecting output to 'testcase-out.txt'", + ",testcase NAME Begin redirecting output to 'testcase-out.txt'", #endif - ".testctrl CMD ... Run various sqlite3_test_control() operations", + ",testctrl CMD ... Run various sqlite3_test_control() operations", " Run \".testctrl\" with no arguments for details", ".timeout MS Try opening locked tables for MS milliseconds", ".timer on|off Turn SQL timer on or off", @@ -19966,16 +20318,41 @@ static int showHelp(FILE *out, const char *zPattern){ || cli_strcmp(zPattern,"-all")==0 || cli_strcmp(zPattern,"--all")==0 ){ - /* Show all commands, but only one line per command */ - if( zPattern==0 ) zPattern = ""; + enum HelpWanted { HW_NoCull = 0, HW_SummaryOnly = 1, HW_Undoc = 2 }; + enum HelpHave { HH_Undoc = 2, HH_Summary = 1, HH_More = 0 }; + /* Show all or most commands + ** *zPattern==0 => summary of documented commands only + ** *zPattern=='0' => whole help for undocumented commands + ** Otherwise => whole help for documented commands + */ + enum HelpWanted hw = HW_SummaryOnly; + enum HelpHave hh = HH_More; + if( zPattern!=0 ){ + hw = (*zPattern=='0')? HW_NoCull|HW_Undoc : HW_NoCull; + } for(i=0; i) of the blob. -*/ -static void shellInt32( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const unsigned char *pBlob; - int nBlob; - int iInt; - - UNUSED_PARAMETER(argc); - nBlob = sqlite3_value_bytes(argv[0]); - pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); - iInt = sqlite3_value_int(argv[1]); - - if( iInt>=0 && (iInt+1)*4<=nBlob ){ - const unsigned char *a = &pBlob[iInt*4]; - sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) - + ((sqlite3_int64)a[1]<<16) - + ((sqlite3_int64)a[2]<< 8) - + ((sqlite3_int64)a[3]<< 0); - sqlite3_result_int64(context, iVal); - } -} - -/* -** Scalar function "shell_idquote(X)" returns string X quoted as an identifier, -** using "..." with internal double-quote characters doubled. -*/ -static void shellIdQuote( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zName = (const char*)sqlite3_value_text(argv[0]); - UNUSED_PARAMETER(argc); - if( zName ){ - char *z = sqlite3_mprintf("\"%w\"", zName); - sqlite3_result_text(context, z, -1, sqlite3_free); - } -} - /* ** Scalar function "usleep(X)" invokes sqlite3_sleep(X) and returns X. */ @@ -20300,97 +20645,6 @@ static void shellUSleepFunc( sqlite3_result_int(context, sleep); } -/* -** Scalar function "shell_escape_crnl" used by the .recover command. -** The argument passed to this function is the output of built-in -** function quote(). If the first character of the input is "'", -** indicating that the value passed to quote() was a text value, -** then this function searches the input for "\n" and "\r" characters -** and adds a wrapper similar to the following: -** -** replace(replace(, '\n', char(10), '\r', char(13)); -** -** Or, if the first character of the input is not "'", then a copy -** of the input is returned. -*/ -static void shellEscapeCrnl( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *zText = (const char*)sqlite3_value_text(argv[0]); - UNUSED_PARAMETER(argc); - if( zText && zText[0]=='\'' ){ - i64 nText = sqlite3_value_bytes(argv[0]); - i64 i; - char zBuf1[20]; - char zBuf2[20]; - const char *zNL = 0; - const char *zCR = 0; - i64 nCR = 0; - i64 nNL = 0; - - for(i=0; zText[i]; i++){ - if( zNL==0 && zText[i]=='\n' ){ - zNL = unused_string(zText, "\\n", "\\012", zBuf1); - nNL = strlen(zNL); - } - if( zCR==0 && zText[i]=='\r' ){ - zCR = unused_string(zText, "\\r", "\\015", zBuf2); - nCR = strlen(zCR); - } - } - - if( zNL || zCR ){ - i64 iOut = 0; - i64 nMax = (nNL > nCR) ? nNL : nCR; - i64 nAlloc = nMax * nText + (nMax+64)*2; - char *zOut = (char*)sqlite3_malloc64(nAlloc); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - return; - } - - if( zNL && zCR ){ - memcpy(&zOut[iOut], "replace(replace(", 16); - iOut += 16; - }else{ - memcpy(&zOut[iOut], "replace(", 8); - iOut += 8; - } - for(i=0; zText[i]; i++){ - if( zText[i]=='\n' ){ - memcpy(&zOut[iOut], zNL, nNL); - iOut += nNL; - }else if( zText[i]=='\r' ){ - memcpy(&zOut[iOut], zCR, nCR); - iOut += nCR; - }else{ - zOut[iOut] = zText[i]; - iOut++; - } - } - - if( zNL ){ - memcpy(&zOut[iOut], ",'", 2); iOut += 2; - memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; - memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; - } - if( zCR ){ - memcpy(&zOut[iOut], ",'", 2); iOut += 2; - memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; - memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; - } - - sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); - sqlite3_free(zOut); - return; - } - } - - sqlite3_result_value(context, argv[0]); -} - /* Flags for open_db(). ** ** The default behavior of open_db() is to exit(1) if the database fails to @@ -20450,11 +20704,29 @@ static void open_db(ShellState *p, int openFlags){ if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ utf8_printf(stderr,"Error: unable to open database \"%s\": %s\n", zDbFilename, sqlite3_errmsg(p->db)); - if( openFlags & OPEN_DB_KEEPALIVE ){ - sqlite3_open(":memory:", &p->db); - return; + if( (openFlags & OPEN_DB_KEEPALIVE)==0 ){ + exit(1); + } + sqlite3_close(p->db); + sqlite3_open(":memory:", &p->db); + if( p->db==0 || SQLITE_OK!=sqlite3_errcode(p->db) ){ + utf8_printf(stderr, + "Also: unable to open substitute in-memory database.\n" + ); + exit(1); + }else{ + utf8_printf(stderr, + "Notice: using substitute in-memory database instead of \"%s\"\n", + zDbFilename); } - exit(1); + } + sqlite3_db_config(p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, (int)0, (int*)0); + + /* Reflect the use or absence of --unsafe-testing invocation. */ + { + int testmode_on = ShellHasFlag(p,SHFLG_TestingMode); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_TRUSTED_SCHEMA, testmode_on,0); + sqlite3_db_config(p->db, SQLITE_DBCONFIG_DEFENSIVE, !testmode_on,0); } #ifndef SQLITE_OMIT_LOAD_EXTENSION @@ -20472,9 +20744,6 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); #endif -#if SQLITE_SHELL_HAVE_RECOVER - sqlite3_dbdata_init(p->db, 0, 0); -#endif #ifdef SQLITE_HAVE_ZLIB if( !p->bSafeModePersist ){ sqlite3_zipfile_init(p->db, 0, 0); @@ -20515,12 +20784,6 @@ static void open_db(ShellState *p, int openFlags){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); - sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, - shellEscapeCrnl, 0, 0); - sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, - shellInt32, 0, 0); - sqlite3_create_function(p->db, "shell_idquote", 1, SQLITE_UTF8, 0, - shellIdQuote, 0, 0); sqlite3_create_function(p->db, "usleep",1,SQLITE_UTF8,0, shellUSleepFunc, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM @@ -20547,9 +20810,9 @@ static void open_db(ShellState *p, int openFlags){ aData = (unsigned char*)readFile(zDbFilename, &nData); }else{ aData = readHexDb(p, &nData); - if( aData==0 ){ - return; - } + } + if( aData==0 ){ + return; } rc = sqlite3_deserialize(p->db, "main", aData, nData, nData, SQLITE_DESERIALIZE_RESIZEABLE | @@ -20563,8 +20826,13 @@ static void open_db(ShellState *p, int openFlags){ } #endif } - if( p->bSafeModePersist && p->db!=0 ){ - sqlite3_set_authorizer(p->db, safeModeAuth, p); + if( p->db!=0 ){ + if( p->bSafeModePersist ){ + sqlite3_set_authorizer(p->db, safeModeAuth, p); + } + sqlite3_db_config( + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 + ); } } @@ -20663,6 +20931,7 @@ static void linenoise_completion(const char *zLine, linenoiseCompletions *lc){ ** \' -> ' ** \\ -> backslash ** \NNN -> ascii character NNN in octal +** \xHH -> ascii character HH in hexadecimal */ static void resolve_backslashes(char *z){ int i, j; @@ -20691,6 +20960,15 @@ static void resolve_backslashes(char *z){ c = '\''; }else if( c=='\\' ){ c = '\\'; + }else if( c=='x' ){ + int nhd = 0, hdv; + u8 hv = 0; + while( nhd<2 && (c=z[i+1+nhd])!=0 && (hdv=hexDigitValue(c))>=0 ){ + hv = (u8)((hv<<4)|hdv); + ++nhd; + } + i += nhd; + c = (u8)hv; }else if( c>='0' && c<='7' ){ c -= '0'; if( z[i+1]>='0' && z[i+1]<='7' ){ @@ -20790,7 +21068,7 @@ static int sql_trace_callback( utf8_printf(p->traceOut, "-- closing database connection\n"); return 0; } - if( mType!=SQLITE_TRACE_ROW && ((const char*)pX)[0]=='-' ){ + if( mType!=SQLITE_TRACE_ROW && pX!=0 && ((const char*)pX)[0]=='-' ){ zSql = (const char*)pX; }else{ pStmt = (sqlite3_stmt*)pP; @@ -20822,7 +21100,7 @@ static int sql_trace_callback( break; } case SQLITE_TRACE_PROFILE: { - sqlite3_int64 nNanosec = *(sqlite3_int64*)pX; + sqlite3_int64 nNanosec = pX ? *(sqlite3_int64*)pX : 0; utf8_printf(p->traceOut, "%.*s; -- %lld ns\n", (int)nSql, zSql, nNanosec); break; } @@ -20898,8 +21176,8 @@ static void import_append_char(ImportCtx *p, int c){ */ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ int c; - int cSep = p->cColSep; - int rSep = p->cRowSep; + int cSep = (u8)p->cColSep; + int rSep = (u8)p->cRowSep; p->n = 0; c = fgetc(p->in); if( c==EOF || seenInterrupt ){ @@ -20988,8 +21266,8 @@ static char *SQLITE_CDECL csv_read_one_field(ImportCtx *p){ */ static char *SQLITE_CDECL ascii_read_one_field(ImportCtx *p){ int c; - int cSep = p->cColSep; - int rSep = p->cRowSep; + int cSep = (u8)p->cColSep; + int rSep = (u8)p->cRowSep; p->n = 0; c = fgetc(p->in); if( c==EOF || seenInterrupt ){ @@ -21183,7 +21461,7 @@ static void tryToCloneSchema( zName = sqlite3_column_text(pQuery, 0); zSql = sqlite3_column_text(pQuery, 1); if( zName==0 || zSql==0 ) continue; - if( sqlite3_stricmp((char*)zName, "sqlite_sequence")!=0 ) continue; + if( sqlite3_stricmp((char*)zName, "sqlite_sequence")==0 ) continue; printf("%s... ", zName); fflush(stdout); sqlite3_exec(newDb, (const char*)zSql, 0, 0, &zErrMsg); if( zErrMsg ){ @@ -21287,7 +21565,7 @@ static int db_int(sqlite3 *db, const char *zSql){ return res; } -#if defined(SQLITE_SHELL_HAVE_RECOVER) +#if SQLITE_SHELL_HAVE_RECOVER /* ** Convert a 2-byte or 4-byte big-endian integer into a native integer */ @@ -21350,7 +21628,9 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){ if( sqlite3_step(pStmt)==SQLITE_ROW && sqlite3_column_bytes(pStmt,0)>100 ){ - memcpy(aHdr, sqlite3_column_blob(pStmt,0), 100); + const u8 *pb = sqlite3_column_blob(pStmt,0); + shell_check_oom(pb); + memcpy(aHdr, pb, 100); sqlite3_finalize(pStmt); }else{ raw_printf(stderr, "unable to read database header\n"); @@ -22167,7 +22447,10 @@ static int arParseCommand( } } } - + if( pAr->eCmd==0 ){ + utf8_printf(stderr, "Required argument missing. Usage:\n"); + return arUsage(stderr); + } return SQLITE_OK; } @@ -22933,7 +23216,7 @@ FROM (\ sqlite3_bind_int(pStmt, 1, nDigits); rc = sqlite3_step(pStmt); sqlite3_finalize(pStmt); - assert(rc==SQLITE_DONE); + if( rc!=SQLITE_DONE ) rc_err_oom_die(SQLITE_NOMEM); } assert(db_int(*pDb, zHasDupes)==0); /* Consider: remove this */ rc = sqlite3_prepare_v2(*pDb, zCollectVar, -1, &pStmt, 0); @@ -23183,7 +23466,6 @@ static int do_meta_command(char *zLine, ShellState *p){ raw_printf(stderr, "Usage: .check GLOB-PATTERN\n"); rc = 2; }else if( (zRes = readFile("testcase-out.txt", 0))==0 ){ - raw_printf(stderr, "Error: cannot read 'testcase-out.txt'\n"); rc = 2; }else if( testcase_glob(azArg[1],zRes)==0 ){ utf8_printf(stderr, @@ -23313,6 +23595,8 @@ static int do_meta_command(char *zLine, ShellState *p){ { "load_extension", SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION }, { "no_ckpt_on_close", SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE }, { "reset_database", SQLITE_DBCONFIG_RESET_DATABASE }, + { "reverse_scanorder", SQLITE_DBCONFIG_REVERSE_SCANORDER }, + { "stmt_scanstatus", SQLITE_DBCONFIG_STMT_SCANSTATUS }, { "trigger_eqp", SQLITE_DBCONFIG_TRIGGER_EQP }, { "trusted_schema", SQLITE_DBCONFIG_TRUSTED_SCHEMA }, { "writable_schema", SQLITE_DBCONFIG_WRITABLE_SCHEMA }, @@ -23864,8 +24148,8 @@ static int do_meta_command(char *zLine, ShellState *p){ " for import\n"); goto meta_command_exit; } - sCtx.cColSep = p->colSeparator[0]; - sCtx.cRowSep = p->rowSeparator[0]; + sCtx.cColSep = (u8)p->colSeparator[0]; + sCtx.cRowSep = (u8)p->rowSeparator[0]; } sCtx.zFile = zFile; sCtx.nLine = 1; @@ -24061,6 +24345,12 @@ static int do_meta_command(char *zLine, ShellState *p){ int isWO = 0; /* True if making an imposter of a WITHOUT ROWID table */ int lenPK = 0; /* Length of the PRIMARY KEY string for isWO tables */ int i; + if( !ShellHasFlag(p,SHFLG_TestingMode) ){ + utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n", + "imposter"); + rc = 1; + goto meta_command_exit; + } if( !(nArg==3 || (nArg==2 && sqlite3_stricmp(azArg[1],"off")==0)) ){ utf8_printf(stderr, "Usage: .imposter INDEX IMPOSTER\n" " .imposter off\n"); @@ -24245,7 +24535,8 @@ static int do_meta_command(char *zLine, ShellState *p){ const char *zFile, *zProc; char *zErrMsg = 0; failIfSafeMode(p, "cannot run .load in safe mode"); - if( nArg<2 ){ + if( nArg<2 || azArg[1][0]==0 ){ + /* Must have a non-empty FILE. (Will not load self.) */ raw_printf(stderr, "Usage: .load FILE ?ENTRYPOINT?\n"); rc = 1; goto meta_command_exit; @@ -24262,19 +24553,25 @@ static int do_meta_command(char *zLine, ShellState *p){ }else #endif -#ifndef SQLITE_SHELL_FIDDLE if( c=='l' && cli_strncmp(azArg[0], "log", n)==0 ){ - failIfSafeMode(p, "cannot run .log in safe mode"); if( nArg!=2 ){ raw_printf(stderr, "Usage: .log FILENAME\n"); rc = 1; }else{ const char *zFile = azArg[1]; + if( p->bSafeMode + && cli_strcmp(zFile,"on")!=0 + && cli_strcmp(zFile,"off")!=0 + ){ + raw_printf(stdout, "cannot set .log to anything other " + "than \"on\" or \"off\"\n"); + zFile = "off"; + } output_file_close(p->pLog); + if( cli_strcmp(zFile,"on")==0 ) zFile = "stdout"; p->pLog = output_file_open(zFile, 0); } }else -#endif if( c=='m' && cli_strncmp(azArg[0], "mode", n)==0 ){ const char *zMode = 0; @@ -24910,6 +25207,10 @@ static int do_meta_command(char *zLine, ShellState *p){ }else{ p->scanstatsOn = (u8)booleanValue(azArg[1]); } + open_db(p, 0); + sqlite3_db_config( + p->db, SQLITE_DBCONFIG_STMT_SCANSTATUS, p->scanstatsOn, (int*)0 + ); #ifndef SQLITE_ENABLE_STMT_SCANSTATUS raw_printf(stderr, "Warning: .scanstats not available in this build.\n"); #endif @@ -25559,28 +25860,36 @@ static int do_meta_command(char *zLine, ShellState *p){ shell_check_oom(zRevText); if( bDebug ) utf8_printf(p->out, "%s\n", zRevText); lrc = sqlite3_prepare_v2(p->db, zRevText, -1, &pStmt, 0); - assert(lrc==SQLITE_OK); - if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); - lrc = SQLITE_ROW==sqlite3_step(pStmt); - if( lrc ){ - const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); - sqlite3_stmt *pCheckStmt; - lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); - if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); - if( SQLITE_OK==lrc ){ - if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){ - double countIrreversible = sqlite3_column_double(pCheckStmt, 0); - if( countIrreversible>0 ){ - int sz = (int)(countIrreversible + 0.5); - utf8_printf(stderr, - "Digest includes %d invalidly encoded text field%s.\n", - sz, (sz>1)? "s": ""); + if( lrc!=SQLITE_OK ){ + /* assert(lrc==SQLITE_NOMEM); // might also be SQLITE_ERROR if the + ** user does cruel and unnatural things like ".limit expr_depth 0". */ + rc = 1; + }else{ + if( zLike ) sqlite3_bind_text(pStmt,1,zLike,-1,SQLITE_STATIC); + lrc = SQLITE_ROW==sqlite3_step(pStmt); + if( lrc ){ + const char *zGenQuery = (char*)sqlite3_column_text(pStmt,0); + sqlite3_stmt *pCheckStmt; + lrc = sqlite3_prepare_v2(p->db, zGenQuery, -1, &pCheckStmt, 0); + if( bDebug ) utf8_printf(p->out, "%s\n", zGenQuery); + if( lrc!=SQLITE_OK ){ + rc = 1; + }else{ + if( SQLITE_ROW==sqlite3_step(pCheckStmt) ){ + double countIrreversible = sqlite3_column_double(pCheckStmt, 0); + if( countIrreversible>0 ){ + int sz = (int)(countIrreversible + 0.5); + utf8_printf(stderr, + "Digest includes %d invalidly encoded text field%s.\n", + sz, (sz>1)? "s": ""); + } } + sqlite3_finalize(pCheckStmt); } - sqlite3_finalize(pCheckStmt); + sqlite3_finalize(pStmt); } - sqlite3_finalize(pStmt); } + if( rc ) utf8_printf(stderr, ".sha3sum failed.\n"); sqlite3_free(zRevText); } #endif /* !defined(*_OMIT_SCHEMA_PRAGMAS) && !defined(*_OMIT_VIRTUALTABLE) */ @@ -25843,6 +26152,12 @@ static int do_meta_command(char *zLine, ShellState *p){ int i, n2; const char *zCmd = 0; + if( !ShellHasFlag(p,SHFLG_TestingMode) ){ + utf8_printf(stderr, ".%s unavailable without --unsafe-testing\n", + "testctrl"); + rc = 1; + goto meta_command_exit; + } open_db(p, 0); zCmd = nArg>=2 ? azArg[1] : "help"; @@ -26790,6 +27105,7 @@ static void process_sqliterc( ** Show available command line options */ static const char zOptions[] = + " -- treat no subsequent arguments as options\n" #if defined(SQLITE_HAVE_ZLIB) && !defined(SQLITE_OMIT_VIRTUALTABLE) " -A ARGS... run \".archive ARGS\" and exit\n" #endif @@ -26841,6 +27157,10 @@ static const char zOptions[] = " -stats print memory stats before each finalize\n" " -table set output mode to 'table'\n" " -tabs set output mode to 'tabs'\n" + " -unsafe-testing allow unsafe commands and modes for testing\n" +#if SHELL_WIN_UTF8_OPT + " -utf8 setup interactive console code page for UTF-8\n" +#endif " -version show SQLite version\n" " -vfs NAME use NAME as the default VFS\n" #ifdef SQLITE_ENABLE_VFSTRACE @@ -26852,9 +27172,9 @@ static const char zOptions[] = ; static void usage(int showDetail){ utf8_printf(stderr, - "Usage: %s [OPTIONS] FILENAME [SQL]\n" + "Usage: %s [OPTIONS] [FILENAME [SQL]]\n" "FILENAME is the name of an SQLite database. A new database is created\n" - "if the file does not previously exist.\n", Argv0); + "if the file does not previously exist. Defaults to :memory:.\n", Argv0); if( showDetail ){ utf8_printf(stderr, "OPTIONS include:\n%s", zOptions); }else{ @@ -26886,9 +27206,11 @@ static void main_init(ShellState *data) { memcpy(data->rowSeparator,SEP_Row, 2); data->showHeader = 0; data->shellFlgs = SHFLG_Lookaside; + sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); +#if !defined(SQLITE_SHELL_FIDDLE) verify_uninitialized(); +#endif sqlite3_config(SQLITE_CONFIG_URI, 1); - sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); @@ -26931,6 +27253,10 @@ static char *cmdline_option_value(int argc, char **argv, int i){ return argv[i]; } +static void sayAbnormalExit(void){ + if( seenInterrupt ) fprintf(stderr, "Program interrupted.\n"); +} + #ifndef SQLITE_SHELL_IS_UTF8 # if (defined(_WIN32) || defined(WIN32)) \ && (defined(_MSC_VER) || (defined(UNICODE) && defined(__GNUC__))) @@ -26945,13 +27271,13 @@ static char *cmdline_option_value(int argc, char **argv, int i){ #endif #if SQLITE_SHELL_IS_UTF8 -int SQLITE_CDECL sqlite3_shell_main(int argc, char **argv){ +int SQLITE_CDECL main(int argc, char **argv){ #else int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char **argv; #endif #ifdef SQLITE_DEBUG - sqlite3_int64 mem_main_enter = sqlite3_memory_used(); + sqlite3_int64 mem_main_enter = 0; #endif char *zErrMsg = 0; #ifdef SQLITE_SHELL_FIDDLE @@ -26965,15 +27291,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ int warnInmemoryDb = 0; int readStdin = 1; int nCmd = 0; + int nOptsEnd = argc; char **azCmd = 0; const char *zVfs = 0; /* Value of -vfs command-line option */ #if !SQLITE_SHELL_IS_UTF8 char **argvToFree = 0; int argcToFree = 0; #endif - - setBinaryMode(stdin, 0); setvbuf(stderr, 0, _IONBF, 0); /* Make sure stderr is unbuffered */ + #ifdef SQLITE_SHELL_FIDDLE stdin_is_interactive = 0; stdout_is_console = 1; @@ -26982,7 +27308,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = isatty(0); stdout_is_console = isatty(1); #endif - +#if SHELL_WIN_UTF8_OPT + atexit(console_restore); /* Needs revision for CLI as library call */ +#endif + atexit(sayAbnormalExit); +#ifdef SQLITE_DEBUG + mem_main_enter = sqlite3_memory_used(); +#endif #if !defined(_WIN32_WCE) if( getenv("SQLITE_DEBUG_BREAK") ){ if( isatty(0) && isatty(2) ){ @@ -27003,6 +27335,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } } #endif + /* Register a valid signal handler early, before much else is done. */ +#ifdef SIGINT + signal(SIGINT, interrupt_handler); +#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) + if( !SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE) ){ + fprintf(stderr, "No ^C handler.\n"); + } +#endif #if USE_SYSTEM_SQLITE+0!=1 if( cli_strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,60)!=0 ){ @@ -27042,15 +27382,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ assert( argc>=1 && argv && argv[0] ); Argv0 = argv[0]; - /* Make sure we have a valid signal handler early, before anything - ** else is done. - */ -#ifdef SIGINT - signal(SIGINT, interrupt_handler); -#elif (defined(_WIN32) || defined(WIN32)) && !defined(_WIN32_WCE) - SetConsoleCtrlHandler(ConsoleCtrlHandler, TRUE); -#endif - #ifdef SQLITE_SHELL_DBNAME_PROC { /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name @@ -27068,11 +27399,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ ** the size of the alternative malloc heap, ** and the first command to execute. */ +#ifndef SQLITE_SHELL_FIDDLE verify_uninitialized(); +#endif for(i=1; inOptsEnd ){ if( data.aAuxDb->zDbFilename==0 ){ data.aAuxDb->zDbFilename = z; }else{ @@ -27084,9 +27417,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ shell_check_oom(azCmd); azCmd[nCmd-1] = z; } + continue; } if( z[1]=='-' ) z++; - if( cli_strcmp(z,"-separator")==0 + if( cli_strcmp(z, "-")==0 ){ + nOptsEnd = i; + continue; + }else if( cli_strcmp(z,"-separator")==0 || cli_strcmp(z,"-nullvalue")==0 || cli_strcmp(z,"-newline")==0 || cli_strcmp(z,"-cmd")==0 @@ -27108,6 +27445,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ zSize = cmdline_option_value(argc, argv, ++i); szHeap = integerValue(zSize); if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000; + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #else (void)cmdline_option_value(argc, argv, ++i); @@ -27121,6 +27459,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( sz>0 && n>0 && 0xffffffffffffLL/sz0 && sz>0) ? malloc(n*sz) : 0, sz, n); data.shellFlgs |= SHFLG_Pagecache; @@ -27130,11 +27469,13 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ if( sz<0 ) sz = 0; n = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( n<0 ) n = 0; + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; }else if( cli_strcmp(z,"-threadsafe")==0 ){ int n; n = (int)integerValue(cmdline_option_value(argc,argv,++i)); + verify_uninitialized(); switch( n ){ case 0: sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); break; case 2: sqlite3_config(SQLITE_CONFIG_MULTITHREAD); break; @@ -27158,10 +27499,12 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif }else if( cli_strcmp(z,"-mmap")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); -#ifdef SQLITE_ENABLE_SORTER_REFERENCES +#if defined(SQLITE_ENABLE_SORTER_REFERENCES) }else if( cli_strcmp(z,"-sorterref")==0 ){ sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + verify_uninitialized(); sqlite3_config(SQLITE_CONFIG_SORTERREF_SIZE, (int)sz); #endif }else if( cli_strcmp(z,"-vfs")==0 ){ @@ -27195,11 +27538,15 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ }else if( cli_strcmp(z,"-nonce")==0 ){ free(data.zNonce); data.zNonce = strdup(argv[++i]); + }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ + ShellSetFlag(&data,SHFLG_TestingMode); }else if( cli_strcmp(z,"-safe")==0 ){ /* no-op - catch this on the second pass */ } } +#ifndef SQLITE_SHELL_FIDDLE verify_uninitialized(); +#endif #ifdef SQLITE_SHELL_INIT_PROC @@ -27263,7 +27610,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ */ for(i=1; i=nOptsEnd ) continue; if( z[1]=='-' ){ z++; } if( cli_strcmp(z,"-init")==0 ){ i++; @@ -27355,6 +27702,10 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ stdin_is_interactive = 1; }else if( cli_strcmp(z,"-batch")==0 ){ stdin_is_interactive = 0; + }else if( cli_strcmp(z,"-utf8")==0 ){ +#if SHELL_WIN_UTF8_OPT + console_utf8 = 1; +#endif /* SHELL_WIN_UTF8_OPT */ }else if( cli_strcmp(z,"-heap")==0 ){ i++; }else if( cli_strcmp(z,"-pagecache")==0 ){ @@ -27425,6 +27776,8 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ #endif }else if( cli_strcmp(z,"-safe")==0 ){ data.bSafeMode = data.bSafeModePersist = 1; + }else if( cli_strcmp(z,"-unsafe-testing")==0 ){ + /* Acted upon in first pass. */ }else{ utf8_printf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); raw_printf(stderr,"Use -help for a list of options.\n"); @@ -27432,6 +27785,14 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } data.cMode = data.mode; } +#if SHELL_WIN_UTF8_OPT + if( console_utf8 && stdin_is_interactive ){ + console_prepare(); + }else{ + setBinaryMode(stdin, 0); + console_utf8 = 0; + } +#endif if( !readStdin ){ /* Run all arguments that do not begin with '-' as if they were separate @@ -27447,6 +27808,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ } }else{ open_db(&data, 0); + echo_group_input(&data, azCmd[i]); rc = shell_exec(&data, azCmd[i], &zErrMsg); if( zErrMsg || rc ){ if( zErrMsg!=0 ){ @@ -27467,7 +27829,6 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char *zHome; char *zHistory; int nHistory; - print_FTL_version(); printf( "SQLite version %s %.19s\n" /*extra-version-info*/ "Enter \".help\" for usage hints.\n", diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index 14520bcf6..dd3b5c575 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.41.1. By combining all the individual C code files into this +** version 3.42.0. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -123,6 +123,10 @@ #define SQLITE_4_BYTE_ALIGNED_MALLOC #endif /* defined(_MSC_VER) && !defined(_WIN64) */ +#if !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 +#define HAVE_LOG2 0 +#endif /* !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 */ + #endif /* SQLITE_MSVC_H */ /************** End of msvc.h ************************************************/ @@ -452,9 +456,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.41.1" -#define SQLITE_VERSION_NUMBER 3041001 -#define SQLITE_SOURCE_ID "2023-03-10 12:13:52 20399f3eda5ec249d147ba9e48da6e87f969d7966a9a896764ca437ff7e737ff" +#define SQLITE_VERSION "3.42.0" +#define SQLITE_VERSION_NUMBER 3042000 +#define SQLITE_SOURCE_ID "2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -1961,20 +1965,23 @@ SQLITE_API int sqlite3_os_end(void); ** must ensure that no other SQLite interfaces are invoked by other ** threads while sqlite3_config() is running. ** -** The sqlite3_config() interface -** may only be invoked prior to library initialization using -** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. -** ^If sqlite3_config() is called after [sqlite3_initialize()] and before -** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. -** Note, however, that ^sqlite3_config() can be called as part of the -** implementation of an application-defined [sqlite3_os_init()]. -** ** The first argument to sqlite3_config() is an integer ** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments ** vary depending on the [configuration option] ** in the first argument. ** +** For most configuration options, the sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** The exceptional configuration options that may be invoked at any time +** are called "anytime configuration options". +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] with a first argument that is not an anytime +** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. @@ -2082,6 +2089,23 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. ** +** Most of the configuration options for sqlite3_config() +** will only work if invoked prior to [sqlite3_initialize()] or after +** [sqlite3_shutdown()]. The few exceptions to this rule are called +** "anytime configuration options". +** ^Calling [sqlite3_config()] with a first argument that is not an +** anytime configuration option in between calls to [sqlite3_initialize()] and +** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE. +** +** The set of anytime configuration options can change (by insertions +** and/or deletions) from one release of SQLite to the next. +** As of SQLite version 3.42.0, the complete set of anytime configuration +** options is: +**
    +**
  • SQLITE_CONFIG_LOG +**
  • SQLITE_CONFIG_PCACHE_HDRSZ +**
+** ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications ** should check the return code from [sqlite3_config()] to make sure that @@ -2428,28 +2452,28 @@ struct sqlite3_mem_methods { ** compile-time option is not set, then the default maximum is 1073741824. ** */ -#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ -#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ -#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ -#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ -#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ -#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ -#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ -#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ -#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ -/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ -#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ -#define SQLITE_CONFIG_PCACHE 14 /* no-op */ -#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ -#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ -#define SQLITE_CONFIG_URI 17 /* int */ -#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ -#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ -#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ -#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ @@ -2684,7 +2708,7 @@ struct sqlite3_mem_methods { ** ** ** [[SQLITE_DBCONFIG_DQS_DML]] -**
SQLITE_DBCONFIG_DQS_DML +**
SQLITE_DBCONFIG_DQS_DML
**
The SQLITE_DBCONFIG_DQS_DML option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DML statements ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The @@ -2693,7 +2717,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_DQS_DDL]] -**
SQLITE_DBCONFIG_DQS_DDL +**
SQLITE_DBCONFIG_DQS_DDL
**
The SQLITE_DBCONFIG_DQS option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DDL statements, ** such as CREATE TABLE and CREATE INDEX. The @@ -2702,7 +2726,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] -**
SQLITE_DBCONFIG_TRUSTED_SCHEMA +**
SQLITE_DBCONFIG_TRUSTED_SCHEMA
**
The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to ** assume that database schemas are untainted by malicious content. ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite @@ -2722,7 +2746,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] -**
SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +**
SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
**
The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly ** created database file to have a schema format version number (the 4-byte @@ -2731,7 +2755,7 @@ struct sqlite3_mem_methods { ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, ** newly created databases are generally not understandable by SQLite versions ** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there -** is now scarcely any need to generated database files that are compatible +** is now scarcely any need to generate database files that are compatible ** all the way back to version 3.0.0, and so this setting is of little ** practical use, but is provided so that SQLite can continue to claim the ** ability to generate new database files that are compatible with version @@ -2742,6 +2766,38 @@ struct sqlite3_mem_methods { ** not considered a bug since SQLite versions 3.3.0 and earlier do not support ** either generated columns or decending indexes. **
+** +** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] +**
SQLITE_DBCONFIG_STMT_SCANSTATUS
+**
The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is set (collection of statistics is enabled) +** by default. This option takes two arguments: an integer and a pointer to +** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the statement scanstatus option. If the second argument +** is not NULL, then the value of the statement scanstatus setting after +** processing the first argument is written into the integer that the second +** argument points to. +**
+** +** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] +**
SQLITE_DBCONFIG_REVERSE_SCANORDER
+**
The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order +** in which tables and indexes are scanned so that the scans start at the end +** and work toward the beginning rather than starting at the beginning and +** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** two arguments which are an integer and a pointer to an integer. The first +** argument is 1, 0, or -1 to enable, disable, or leave unchanged the +** reverse scan order flag, respectively. If the second argument is not NULL, +** then 0 or 1 is written into the integer that the second argument points to +** depending on if the reverse scan order flag is set after processing the +** first argument. +**
+** ** */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -2762,7 +2818,9 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ +#define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -6507,6 +6565,13 @@ SQLITE_API void sqlite3_activate_cerod( ** of the default VFS is not implemented correctly, or not implemented at ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. +** +** If a negative argument is passed to sqlite3_sleep() the results vary by +** VFS and operating system. Some system treat a negative argument as an +** instruction to sleep forever. Others understand it to mean do not sleep +** at all. ^In SQLite version 3.42.0 and later, a negative +** argument passed into sqlite3_sleep() is changed to zero before it is relayed +** down into the xSleep method of the VFS. */ SQLITE_API int sqlite3_sleep(int); @@ -8134,9 +8199,9 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** is undefined if the mutex is not currently entered by the ** calling thread or is not currently allocated. ** -** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or -** sqlite3_mutex_leave() is a NULL pointer, then all three routines -** behave as no-ops. +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), +** sqlite3_mutex_leave(), or sqlite3_mutex_free() is a NULL pointer, +** then any of the four routines behaves as a no-op. ** ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. */ @@ -9870,18 +9935,28 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]
SQLITE_VTAB_INNOCUOUS
**
Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a ** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS ** flag unless absolutely necessary. **
+** +** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]
SQLITE_VTAB_USES_ALL_SCHEMAS
+**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** instruct the query planner to begin at least a read transaction on +** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the +** virtual table is used. +**
** */ #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 #define SQLITE_VTAB_INNOCUOUS 2 #define SQLITE_VTAB_DIRECTONLY 3 +#define SQLITE_VTAB_USES_ALL_SCHEMAS 4 /* ** CAPI3REF: Determine The Virtual Table Conflict Policy @@ -11056,16 +11131,20 @@ SQLITE_API int sqlite3session_create( SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); /* -** CAPIREF: Conigure a Session Object +** CAPI3REF: Configure a Session Object ** METHOD: sqlite3_session ** ** This method is used to configure a session object after it has been -** created. At present the only valid value for the second parameter is -** [SQLITE_SESSION_OBJCONFIG_SIZE]. +** created. At present the only valid values for the second parameter are +** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID]. ** -** Arguments for sqlite3session_object_config() +*/ +SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); + +/* +** CAPI3REF: Options for sqlite3session_object_config ** -** The following values may passed as the the 4th parameter to +** The following values may passed as the the 2nd parameter to ** sqlite3session_object_config(). ** **
SQLITE_SESSION_OBJCONFIG_SIZE
@@ -11081,12 +11160,21 @@ SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); ** ** It is an error (SQLITE_MISUSE) to attempt to modify this setting after ** the first table has been attached to the session object. +** +**
SQLITE_SESSION_OBJCONFIG_ROWID
+** This option is used to set, clear or query the flag that enables +** collection of data for tables with no explicit PRIMARY KEY. +** +** Normally, tables with no explicit PRIMARY KEY are simply ignored +** by the sessions module. However, if this flag is set, it behaves +** as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted +** as their leftmost columns. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. */ -SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); - -/* -*/ -#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_ROWID 2 /* ** CAPI3REF: Enable Or Disable A Session Object @@ -12219,9 +12307,23 @@ SQLITE_API int sqlite3changeset_apply_v2( ** Invert the changeset before applying it. This is equivalent to inverting ** a changeset using sqlite3changeset_invert() before applying it. It is ** an error to specify this flag with a patchset. +** +**
SQLITE_CHANGESETAPPLY_IGNORENOOP
+** Do not invoke the conflict handler callback for any changes that +** would not actually modify the database even if they were applied. +** Specifically, this means that the conflict handler is not invoked +** for: +**
    +**
  • a delete change if the row being deleted cannot be found, +**
  • an update change if the modified fields are already set to +** their new values in the conflicting row, or +**
  • an insert change if all fields of the conflicting row match +** the row being inserted. +**
*/ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 #define SQLITE_CHANGESETAPPLY_INVERT 0x0002 +#define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 /* ** CAPI3REF: Constants Passed To The Conflict Handler @@ -13518,8 +13620,8 @@ struct fts5_api { #endif /* -** WAL mode depends on atomic aligned 32-bit loads and stores in a few -** places. The following macros try to make this explicit. +** A few places in the code require atomic load/store of aligned +** integer values. */ #ifndef __has_extension # define __has_extension(x) 0 /* compatibility with non-clang compilers */ @@ -13575,15 +13677,22 @@ struct fts5_api { #endif /* -** A macro to hint to the compiler that a function should not be +** Macros to hint to the compiler that a function should or should not be ** inlined. */ #if defined(__GNUC__) # define SQLITE_NOINLINE __attribute__((noinline)) +# define SQLITE_INLINE __attribute__((always_inline)) inline #elif defined(_MSC_VER) && _MSC_VER>=1310 # define SQLITE_NOINLINE __declspec(noinline) +# define SQLITE_INLINE __forceinline #else # define SQLITE_NOINLINE +# define SQLITE_INLINE +#endif +#if defined(SQLITE_COVERAGE_TEST) || defined(__STRICT_ANSI__) +# undef SQLITE_INLINE +# define SQLITE_INLINE #endif /* @@ -16544,6 +16653,10 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int); SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); #endif +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr); +#endif + #endif /* SQLITE_VDBE_H */ /************** End of vdbe.h ************************************************/ @@ -16592,7 +16705,7 @@ struct PgHdr { ** private to pcache.c and should not be accessed by other modules. ** pCache is grouped with the public elements for efficiency. */ - i16 nRef; /* Number of users of this page */ + i64 nRef; /* Number of users of this page */ PgHdr *pDirtyNext; /* Next element in list of dirty pages */ PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ /* NB: pDirtyNext and pDirtyPrev are undefined if the @@ -16673,12 +16786,12 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *); SQLITE_PRIVATE void sqlite3PcacheClear(PCache*); /* Return the total number of outstanding page references */ -SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache*); +SQLITE_PRIVATE i64 sqlite3PcacheRefCount(PCache*); /* Increment the reference count of an existing page */ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr*); -SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr*); +SQLITE_PRIVATE i64 sqlite3PcachePageRefcount(PgHdr*); /* Return the total number of pages stored in the cache */ SQLITE_PRIVATE int sqlite3PcachePagecount(PCache*); @@ -17253,7 +17366,7 @@ struct sqlite3 { #define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ /* result set is empty */ #define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommit 0x00000400 /* READ UNCOMMITTED in shared-cache */ +#define SQLITE_StmtScanStatus 0x00000400 /* Enable stmt_scanstats() counters */ #define SQLITE_NoCkptOnClose 0x00000800 /* No checkpoint on close()/DETACH */ #define SQLITE_ReverseOrder 0x00001000 /* Reverse unordered SELECTs */ #define SQLITE_RecTriggers 0x00002000 /* Enable recursive triggers */ @@ -17279,6 +17392,7 @@ struct sqlite3 { /* DELETE, or UPDATE and return */ /* the count using a callback. */ #define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ +#define SQLITE_ReadUncommit HI(0x00004) /* READ UNCOMMITTED in shared-cache */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG @@ -17335,6 +17449,7 @@ struct sqlite3 { /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ #define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ #define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ +#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ #define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* @@ -17806,6 +17921,7 @@ struct VTable { sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ u8 bConstraint; /* True if constraints are supported */ + u8 bAllSchemas; /* True if might use any attached schema */ u8 eVtabRisk; /* Riskiness of allowing hacker access */ int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ @@ -18186,6 +18302,7 @@ struct Index { ** expression, or a reference to a VIRTUAL column */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ + int mxSample; /* Number of slots allocated to aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ @@ -18837,7 +18954,7 @@ struct NameContext { #define NC_HasAgg 0x000010 /* One or more aggregate functions seen */ #define NC_IdxExpr 0x000020 /* True if resolving columns of CREATE INDEX */ #define NC_SelfRef 0x00002e /* Combo: PartIdx, isCheck, GenCol, and IdxExpr */ -#define NC_VarSelect 0x000040 /* A correlated subquery has been seen */ +#define NC_Subquery 0x000040 /* A subquery has been seen */ #define NC_UEList 0x000080 /* True if uNC.pEList is used */ #define NC_UAggInfo 0x000100 /* True if uNC.pAggInfo is used */ #define NC_UUpsert 0x000200 /* True if uNC.pUpsert is used */ @@ -19208,6 +19325,9 @@ struct Parse { u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ +#endif +#ifdef SQLITE_DEBUG + u8 ifNotExists; /* Might be true if IF NOT EXISTS. Assert()s only */ #endif int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ @@ -19669,6 +19789,7 @@ struct Walker { struct CoveringIndexCheck *pCovIdxCk; /* Check for covering index */ SrcItem *pSrcItem; /* A single FROM clause item */ DbFixer *pFix; /* See sqlite3FixSelect() */ + Mem *aMem; /* See sqlite3BtreeCursorHint() */ } u; }; @@ -19938,6 +20059,8 @@ SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) # define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) # define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) +# define sqlite3JsonId1(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x42) +# define sqlite3JsonId2(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x46) #else # define sqlite3Toupper(x) toupper((unsigned char)(x)) # define sqlite3Isspace(x) isspace((unsigned char)(x)) @@ -19947,6 +20070,8 @@ SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) # define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') +# define sqlite3JsonId1(x) (sqlite3IsIdChar(x)&&(x)<'0') +# define sqlite3JsonId2(x) sqlite3IsIdChar(x) #endif SQLITE_PRIVATE int sqlite3IsIdChar(u8); @@ -20140,6 +20265,10 @@ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int); SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int); SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int); SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*); +SQLITE_PRIVATE void sqlite3TouchRegister(Parse*,int); +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse*,int); +#endif #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int); #endif @@ -20290,7 +20419,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList Expr*,ExprList*,u32,Expr*); SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); -SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, Trigger*); SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) SQLITE_PRIVATE Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*); @@ -20379,7 +20508,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); -SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr*,const SrcItem*); +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); #endif @@ -20827,10 +20956,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3*, int, const char *); SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *, VTable *); SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); -#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ - && !defined(SQLITE_OMIT_VIRTUALTABLE) -SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(sqlite3_index_info*); -#endif +SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse*); SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); @@ -21077,6 +21203,12 @@ SQLITE_PRIVATE int sqlite3KvvfsInit(void); SQLITE_PRIVATE sqlite3_uint64 sqlite3Hwtime(void); #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define IS_STMT_SCANSTATUS(db) (db->flags & SQLITE_StmtScanStatus) +#else +# define IS_STMT_SCANSTATUS(db) 0 +#endif + #endif /* SQLITEINT_H */ /************** End of sqliteInt.h *******************************************/ @@ -22072,7 +22204,7 @@ SQLITE_PRIVATE const unsigned char *sqlite3aGTb = &sqlite3UpperToLower[256+12-OP ** isalnum() 0x06 ** isxdigit() 0x08 ** toupper() 0x20 -** SQLite identifier character 0x40 +** SQLite identifier character 0x40 $, _, or non-ascii ** Quote character 0x80 ** ** Bit 0x20 is set if the mapped character requires translation to upper @@ -22266,7 +22398,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ 0, /* iPrngSeed */ #ifdef SQLITE_DEBUG - {0,0,0,0,0,0} /* aTune */ + {0,0,0,0,0,0}, /* aTune */ #endif }; @@ -23565,6 +23697,7 @@ struct DateTime { char validTZ; /* True (1) if tz is valid */ char tzSet; /* Timezone was set explicitly */ char isError; /* An overflow has occurred */ + char useSubsec; /* Display subsecond precision */ }; @@ -23879,6 +24012,11 @@ static int parseDateOrTime( }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ setRawDateNumber(p, r); return 0; + }else if( (sqlite3StrICmp(zDate,"subsec")==0 + || sqlite3StrICmp(zDate,"subsecond")==0) + && sqlite3NotPureFunc(context) ){ + p->useSubsec = 1; + return setDateTimeToCurrent(context, p); } return 1; } @@ -24293,8 +24431,22 @@ static int parseModifier( ** ** Move the date backwards to the beginning of the current day, ** or month or year. + ** + ** subsecond + ** subsec + ** + ** Show subsecond precision in the output of datetime() and + ** unixepoch() and strftime('%s'). */ - if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break; + if( sqlite3_strnicmp(z, "start of ", 9)!=0 ){ + if( sqlite3_stricmp(z, "subsec")==0 + || sqlite3_stricmp(z, "subsecond")==0 + ){ + p->useSubsec = 1; + rc = 0; + } + break; + } if( !p->validJD && !p->validYMD && !p->validHMS ) break; z += 9; computeYMD(p); @@ -24492,7 +24644,11 @@ static void unixepochFunc( DateTime x; if( isDate(context, argc, argv, &x)==0 ){ computeJD(&x); - sqlite3_result_int64(context, x.iJD/1000 - 21086676*(i64)10000); + if( x.useSubsec ){ + sqlite3_result_double(context, (x.iJD - 21086676*(i64)10000000)/1000.0); + }else{ + sqlite3_result_int64(context, x.iJD/1000 - 21086676*(i64)10000); + } } } @@ -24508,8 +24664,8 @@ static void datetimeFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - int Y, s; - char zBuf[24]; + int Y, s, n; + char zBuf[32]; computeYMD_HMS(&x); Y = x.Y; if( Y<0 ) Y = -Y; @@ -24530,15 +24686,28 @@ static void datetimeFunc( zBuf[15] = '0' + (x.m/10)%10; zBuf[16] = '0' + (x.m)%10; zBuf[17] = ':'; - s = (int)x.s; - zBuf[18] = '0' + (s/10)%10; - zBuf[19] = '0' + (s)%10; - zBuf[20] = 0; + if( x.useSubsec ){ + s = (int)1000.0*x.s; + zBuf[18] = '0' + (s/10000)%10; + zBuf[19] = '0' + (s/1000)%10; + zBuf[20] = '.'; + zBuf[21] = '0' + (s/100)%10; + zBuf[22] = '0' + (s/10)%10; + zBuf[23] = '0' + (s)%10; + zBuf[24] = 0; + n = 24; + }else{ + s = (int)x.s; + zBuf[18] = '0' + (s/10)%10; + zBuf[19] = '0' + (s)%10; + zBuf[20] = 0; + n = 20; + } if( x.Y<0 ){ zBuf[0] = '-'; - sqlite3_result_text(context, zBuf, 20, SQLITE_TRANSIENT); + sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT); }else{ - sqlite3_result_text(context, &zBuf[1], 19, SQLITE_TRANSIENT); + sqlite3_result_text(context, &zBuf[1], n-1, SQLITE_TRANSIENT); } } } @@ -24555,7 +24724,7 @@ static void timeFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - int s; + int s, n; char zBuf[16]; computeHMS(&x); zBuf[0] = '0' + (x.h/10)%10; @@ -24564,11 +24733,24 @@ static void timeFunc( zBuf[3] = '0' + (x.m/10)%10; zBuf[4] = '0' + (x.m)%10; zBuf[5] = ':'; - s = (int)x.s; - zBuf[6] = '0' + (s/10)%10; - zBuf[7] = '0' + (s)%10; - zBuf[8] = 0; - sqlite3_result_text(context, zBuf, 8, SQLITE_TRANSIENT); + if( x.useSubsec ){ + s = (int)1000.0*x.s; + zBuf[6] = '0' + (s/10000)%10; + zBuf[7] = '0' + (s/1000)%10; + zBuf[8] = '.'; + zBuf[9] = '0' + (s/100)%10; + zBuf[10] = '0' + (s/10)%10; + zBuf[11] = '0' + (s)%10; + zBuf[12] = 0; + n = 12; + }else{ + s = (int)x.s; + zBuf[6] = '0' + (s/10)%10; + zBuf[7] = '0' + (s)%10; + zBuf[8] = 0; + n = 8; + } + sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT); } } @@ -24699,8 +24881,13 @@ static void strftimeFunc( break; } case 's': { - i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000); - sqlite3_str_appendf(&sRes,"%lld",iS); + if( x.useSubsec ){ + sqlite3_str_appendf(&sRes,"%.3f", + (x.iJD - 21086676*(i64)10000000)/1000.0); + }else{ + i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000); + sqlite3_str_appendf(&sRes,"%lld",iS); + } break; } case 'S': { @@ -30071,6 +30258,20 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** "*val" is a u64. *msd is a divisor used to extract the +** most significant digit of *val. Extract that most significant +** digit and return it. +*/ +static char et_getdigit_int(u64 *val, u64 *msd){ + u64 x = (*val)/(*msd); + *val -= x*(*msd); + if( *msd>=10 ) *msd /= 10; + return '0' + (char)(x & 15); +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + /* ** Set the StrAccum object to an error mode. */ @@ -30163,6 +30364,8 @@ SQLITE_API void sqlite3_str_vappendf( char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ sqlite_uint64 longvalue; /* Value for integer types */ LONGDOUBLE_TYPE realvalue; /* Value for real types */ + sqlite_uint64 msd; /* Divisor to get most-significant-digit + ** of longvalue */ const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ @@ -30469,52 +30672,78 @@ SQLITE_API void sqlite3_str_vappendf( }else{ prefix = flag_prefix; } + exp = 0; if( xtype==etGENERIC && precision>0 ) precision--; testcase( precision>0xfff ); - idx = precision & 0xfff; - rounder = arRound[idx%10]; - while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } - if( xtype==etFLOAT ){ - double rx = (double)realvalue; - sqlite3_uint64 u; - int ex; - memcpy(&u, &rx, sizeof(u)); - ex = -1023 + (int)((u>>52)&0x7ff); - if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; - realvalue += rounder; - } - /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ - exp = 0; - if( sqlite3IsNaN((double)realvalue) ){ - bufpt = "NaN"; - length = 3; - break; - } - if( realvalue>0.0 ){ - LONGDOUBLE_TYPE scale = 1.0; - while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } - while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } - realvalue /= scale; - while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } - while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } - if( exp>350 ){ - bufpt = buf; - buf[0] = prefix; - memcpy(buf+(prefix!=0),"Inf",4); - length = 3+(prefix!=0); + if( realvalue<1.0e+16 + && realvalue==(LONGDOUBLE_TYPE)(longvalue = (u64)realvalue) + ){ + /* Number is a pure integer that can be represented as u64 */ + for(msd=1; msd*10<=longvalue; msd *= 10, exp++){} + if( exp>precision && xtype!=etFLOAT ){ + u64 rnd = msd/2; + int kk = precision; + while( kk-- > 0 ){ rnd /= 10; } + longvalue += rnd; + } + }else{ + msd = 0; + longvalue = 0; /* To prevent a compiler warning */ + idx = precision & 0xfff; + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ + double rx = (double)realvalue; + sqlite3_uint64 u; + int ex; + memcpy(&u, &rx, sizeof(u)); + ex = -1023 + (int)((u>>52)&0x7ff); + if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; + realvalue += rounder; + } + if( sqlite3IsNaN((double)realvalue) ){ + if( flag_zeropad ){ + bufpt = "null"; + length = 4; + }else{ + bufpt = "NaN"; + length = 3; + } break; } + + /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ + if( ALWAYS(realvalue>0.0) ){ + LONGDOUBLE_TYPE scale = 1.0; + while( realvalue>=1e100*scale && exp<=350){ scale*=1e100;exp+=100;} + while( realvalue>=1e10*scale && exp<=350 ){ scale*=1e10; exp+=10; } + while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } + realvalue /= scale; + while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } + while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } + if( exp>350 ){ + if( flag_zeropad ){ + realvalue = 9.0; + exp = 999; + }else{ + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); + break; + } + } + if( xtype!=etFLOAT ){ + realvalue += rounder; + if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } + } + } } - bufpt = buf; + /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ - if( xtype!=etFLOAT ){ - realvalue += rounder; - if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } - } if( xtype==etGENERIC ){ flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ @@ -30531,16 +30760,18 @@ SQLITE_API void sqlite3_str_vappendf( }else{ e2 = exp; } + nsd = 16 + flag_altform2*10; + bufpt = buf; { i64 szBufNeeded; /* Size of a temporary buffer needed */ szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; + if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; if( szBufNeeded > etBUFSIZE ){ bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); if( bufpt==0 ) return; } } zOut = bufpt; - nsd = 16 + flag_altform2*10; flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ if( prefix ){ @@ -30549,9 +30780,15 @@ SQLITE_API void sqlite3_str_vappendf( /* Digits prior to the decimal point */ if( e2<0 ){ *(bufpt++) = '0'; + }else if( msd>0 ){ + for(; e2>=0; e2--){ + *(bufpt++) = et_getdigit_int(&longvalue,&msd); + if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ','; + } }else{ for(; e2>=0; e2--){ *(bufpt++) = et_getdigit(&realvalue,&nsd); + if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ','; } } /* The decimal point */ @@ -30565,8 +30802,14 @@ SQLITE_API void sqlite3_str_vappendf( *(bufpt++) = '0'; } /* Significant digits after the decimal point */ - while( (precision--)>0 ){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); + if( msd>0 ){ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit_int(&longvalue,&msd); + } + }else{ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } } /* Remove trailing zeros and the "." if no digits follow the "." */ if( flag_rtz && flag_dp ){ @@ -31247,12 +31490,22 @@ SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_li return zBuf; } SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ - char *z; + StrAccum acc; va_list ap; + if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf ) zBuf[0] = 0; + return zBuf; + } +#endif + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); va_start(ap,zFormat); - z = sqlite3_vsnprintf(n, zBuf, zFormat, ap); + sqlite3_str_vappendf(&acc, zFormat, ap); va_end(ap); - return z; + zBuf[acc.nChar] = 0; + return zBuf; } /* @@ -34282,13 +34535,15 @@ SQLITE_PRIVATE int sqlite3Int64ToText(i64 v, char *zOut){ } i = sizeof(zTemp)-2; zTemp[sizeof(zTemp)-1] = 0; - do{ - zTemp[i--] = (x%10) + '0'; + while( 1 /*exit-by-break*/ ){ + zTemp[i] = (x%10) + '0'; x = x/10; - }while( x ); - if( v<0 ) zTemp[i--] = '-'; - memcpy(zOut, &zTemp[i+1], sizeof(zTemp)-1-i); - return sizeof(zTemp)-2-i; + if( x==0 ) break; + i--; + }; + if( v<0 ) zTemp[--i] = '-'; + memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); + return sizeof(zTemp)-1-i; } /* @@ -34453,7 +34708,9 @@ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ u = u*16 + sqlite3HexToInt(z[k]); } memcpy(pOut, &u, 8); - return (z[k]==0 && k-i<=16) ? 0 : 2; + if( k-i>16 ) return 2; + if( z[k]!=0 ) return 1; + return 0; }else #endif /* SQLITE_OMIT_HEX_INTEGER */ { @@ -34489,7 +34746,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ u32 u = 0; zNum += 2; while( zNum[0]=='0' ) zNum++; - for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + for(i=0; i<8 && sqlite3Isxdigit(zNum[i]); i++){ u = u*16 + sqlite3HexToInt(zNum[i]); } if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ @@ -36985,7 +37242,7 @@ SQLITE_PRIVATE int sqlite3KvvfsInit(void){ #endif /* Use pread() and pwrite() if they are available */ -#if defined(__APPLE__) +#if defined(__APPLE__) || defined(__linux__) # define HAVE_PREAD 1 # define HAVE_PWRITE 1 #endif @@ -40235,12 +40492,6 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ ** Seek to the offset passed as the second argument, then read cnt ** bytes into pBuf. Return the number of bytes actually read. ** -** NB: If you define USE_PREAD or USE_PREAD64, then it might also -** be necessary to define _XOPEN_SOURCE to be 500. This varies from -** one system to another. Since SQLite does not define USE_PREAD -** in any form by default, we will not attempt to define _XOPEN_SOURCE. -** See tickets #2741 and #2681. -** ** To avoid stomping the errno value on a failed read the lastErrno value ** is set before returning. */ @@ -50267,7 +50518,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -50284,7 +50535,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -50304,7 +50555,7 @@ static int winOpen( if( isReadWrite ){ int rc2, isRO = 0; sqlite3BeginBenignMalloc(); - rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); sqlite3EndBenignMalloc(); if( rc2==SQLITE_OK && isRO ) break; } @@ -50527,6 +50778,13 @@ static int winAccess( OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", zFilename, flags, pResOut)); + if( zFilename==0 ){ + *pResOut = 0; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); + return SQLITE_OK; + } + zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); @@ -52654,7 +52912,7 @@ SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ struct PCache { PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ PgHdr *pSynced; /* Last synced page in dirty page list */ - int nRefSum; /* Sum of ref counts over all pages */ + i64 nRefSum; /* Sum of ref counts over all pages */ int szCache; /* Configured cache size */ int szSpill; /* Size before spilling occurs */ int szPage; /* Size of every page in this cache */ @@ -52683,11 +52941,15 @@ struct PCache { PgHdr *pPg; unsigned char *a; int j; - pPg = (PgHdr*)pLower->pExtra; - printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); - a = (unsigned char *)pLower->pBuf; - for(j=0; j<12; j++) printf("%02x", a[j]); - printf(" ptr %p\n", pPg); + if( pLower==0 ){ + printf("%3d: NULL\n", i); + }else{ + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2lld flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf(" ptr %p\n", pPg); + } } static void pcacheDump(PCache *pCache){ int N; @@ -52700,9 +52962,8 @@ struct PCache { if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; for(i=1; i<=N; i++){ pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); - if( pLower==0 ) continue; pcachePageTrace(i, pLower); - if( ((PgHdr*)pLower)->pPage==0 ){ + if( pLower && ((PgHdr*)pLower)->pPage==0 ){ sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); } } @@ -53428,14 +53689,14 @@ SQLITE_PRIVATE PgHdr *sqlite3PcacheDirtyList(PCache *pCache){ ** This is not the total number of pages referenced, but the sum of the ** reference count for all pages. */ -SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ +SQLITE_PRIVATE i64 sqlite3PcacheRefCount(PCache *pCache){ return pCache->nRefSum; } /* ** Return the number of references to the page supplied as an argument. */ -SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr *p){ +SQLITE_PRIVATE i64 sqlite3PcachePageRefcount(PgHdr *p){ return p->nRef; } @@ -58090,6 +58351,8 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; assert( pPager->eState!=PAGER_ERROR ); assert( pPager->eState!=PAGER_READER ); + PAGERTRACE(("Truncate %d npage %u\n", PAGERID(pPager), nPage)); + if( isOpen(pPager->fd) && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) @@ -61007,6 +61270,10 @@ static int getPageNormal( if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; + if( pgno<=pPager->dbSize ){ + sqlite3PcacheRelease(pPg); + pPg = 0; + } goto pager_acquire_err; } if( noContent ){ @@ -61171,10 +61438,12 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ /* ** Release a page reference. ** -** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be -** used if we know that the page being released is not the last page. +** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be used +** if we know that the page being released is not the last reference to page1. ** The btree layer always holds page1 open until the end, so these first -** to routines can be used to release any page other than BtShared.pPage1. +** two routines can be used to release any page other than BtShared.pPage1. +** The assert() at tag-20230419-2 proves that this constraint is always +** honored. ** ** Use sqlite3PagerUnrefPageOne() to release page1. This latter routine ** checks the total number of outstanding pages and if the number of @@ -61190,7 +61459,7 @@ SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage *pPg){ sqlite3PcacheRelease(pPg); } /* Do not use this routine to release the last reference to page1 */ - assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); /* tag-20230419-2 */ } SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ if( pPg ) sqlite3PagerUnrefNotNull(pPg); @@ -62950,13 +63219,15 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ */ static int pagerExclusiveLock(Pager *pPager){ int rc; /* Return code */ + u8 eOrigLock; /* Original lock */ - assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + assert( pPager->eLock>=SHARED_LOCK ); + eOrigLock = pPager->eLock; rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ /* If the attempt to grab the exclusive lock failed, release the ** pending lock that may have been obtained instead. */ - pagerUnlockDb(pPager, SHARED_LOCK); + pagerUnlockDb(pPager, eOrigLock); } return rc; @@ -63961,19 +64232,40 @@ static void walChecksumBytes( assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); assert( nByte<=65536 ); + assert( nByte%4==0 ); - if( nativeCksum ){ + if( !nativeCksum ){ do { + s1 += BYTESWAP32(aData[0]) + s2; + s2 += BYTESWAP32(aData[1]) + s1; + aData += 2; + }while( aDataszPage==szPage ); + if( (int)pWal->szPage!=szPage ){ + return SQLITE_CORRUPT_BKPT; /* TH3 test case: cov1/corrupt155.test */ + } /* Setup information needed to write frames into the WAL */ w.pWal = pWal; @@ -67564,7 +67858,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** byte are used. The integer consists of all bytes that have bit 8 set and ** the first byte with bit 8 clear. The most significant byte of the integer ** appears first. A variable-length integer may not be more than 9 bytes long. -** As a special case, all 8 bytes of the 9th byte are used as data. This +** As a special case, all 8 bits of the 9th byte are used as data. This ** allows a 64-bit integer to be encoded in 9 bytes. ** ** 0x00 becomes 0x00000000 @@ -67948,7 +68242,7 @@ struct BtCursor { #define BTCF_WriteFlag 0x01 /* True if a write cursor */ #define BTCF_ValidNKey 0x02 /* True if info.nKey is valid */ #define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ -#define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ +#define BTCF_AtLast 0x08 /* Cursor is pointing to the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ #define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ #define BTCF_Pinned 0x40 /* Cursor is busy and cannot be moved */ @@ -68093,8 +68387,9 @@ struct IntegrityCk { int rc; /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */ u32 nStep; /* Number of steps into the integrity_check process */ const char *zPfx; /* Error message prefix */ - Pgno v1; /* Value for first %u substitution in zPfx */ - int v2; /* Value for second %d substitution in zPfx */ + Pgno v0; /* Value for first %u substitution in zPfx (root page) */ + Pgno v1; /* Value for second %u substitution in zPfx (current pg) */ + int v2; /* Value for third %d substitution in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ sqlite3 *db; /* Database connection running the check */ @@ -68557,8 +68852,8 @@ SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){ int corruptPageError(int lineno, MemPage *p){ char *zMsg; sqlite3BeginBenignMalloc(); - zMsg = sqlite3_mprintf("database corruption page %d of %s", - (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) + zMsg = sqlite3_mprintf("database corruption page %u of %s", + p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) ); sqlite3EndBenignMalloc(); if( zMsg ){ @@ -69367,8 +69662,25 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow) */ SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ /* Used only by system that substitute their own storage engine */ +#ifdef SQLITE_DEBUG + if( ALWAYS(eHintType==BTREE_HINT_RANGE) ){ + va_list ap; + Expr *pExpr; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3CursorRangeHintExprCheck; + va_start(ap, eHintType); + pExpr = va_arg(ap, Expr*); + w.u.aMem = va_arg(ap, Mem*); + va_end(ap); + assert( pExpr!=0 ); + assert( w.u.aMem!=0 ); + sqlite3WalkExpr(&w, pExpr); + } +#endif /* SQLITE_DEBUG */ } -#endif +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + /* ** Provide flag hints to the cursor. @@ -69453,7 +69765,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ - TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + TRACE(("PTRMAP_UPDATE: %u->(%u,%u)\n", key, eType, parent)); *pRC= rc = sqlite3PagerWrite(pDbPage); if( rc==SQLITE_OK ){ pPtrmap[offset] = eType; @@ -69652,27 +69964,31 @@ static void btreeParseCellPtr( iKey = *pIter; if( iKey>=0x80 ){ u8 x; - iKey = ((iKey&0x7f)<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x =*++pIter) & 0x7f); + iKey = (iKey<<7) ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x10204000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<7) | ((x = *++pIter) & 0x7f); + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); if( x>=0x80 ){ - iKey = (iKey<<8) | (*++pIter); + iKey = (iKey<<8) ^ 0x8000 ^ (*++pIter); } } } } } + }else{ + iKey ^= 0x204000; } + }else{ + iKey ^= 0x4000; } } pIter++; @@ -69749,10 +70065,11 @@ static void btreeParseCell( ** ** cellSizePtrNoPayload() => table internal nodes ** cellSizePtrTableLeaf() => table leaf nodes -** cellSizePtr() => all index nodes & table leaf nodes +** cellSizePtr() => index internal nodes +** cellSizeIdxLeaf() => index leaf nodes */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ u8 *pEnd; /* End mark for a varint */ u32 nSize; /* Size value to return */ @@ -69765,6 +70082,49 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pPage->xParseCell(pPage, pCell, &debuginfo); #endif + assert( pPage->childPtrSize==4 ); + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pItermaxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + assert( nSize>4 ); + }else{ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4 + (u16)(pIter - pCell); + } + assert( nSize==debuginfo.nSize || CORRUPT_DB ); + return (u16)nSize; +} +static u16 cellSizePtrIdxLeaf(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#endif + + assert( pPage->childPtrSize==0 ); nSize = *pIter; if( nSize>=0x80 ){ pEnd = &pIter[8]; @@ -70001,10 +70361,10 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ /* These conditions have already been verified in btreeInitPage() ** if PRAGMA cell_size_check=ON. */ - if( pciCellLast ){ + if( pc>iCellLast ){ return SQLITE_CORRUPT_PAGE(pPage); } - assert( pc>=iCellStart && pc<=iCellLast ); + assert( pc>=0 && pc<=iCellLast ); size = pPage->xCellSize(pPage, &src[pc]); cbrk -= size; if( cbrkusableSize ){ @@ -70119,7 +70479,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** allocation is being made in order to insert a new cell, so we will ** also end up needing a new cell pointer. */ -static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ +static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ int top; /* First byte of cell content area */ @@ -70145,13 +70505,14 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** integer, so a value of 0 is used in its place. */ pTmp = &data[hdr+5]; top = get2byte(pTmp); - assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ if( gap>top ){ if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536; }else{ return SQLITE_CORRUPT_PAGE(pPage); } + }else if( top>(int)pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); } /* If there is enough space between gap and top for one more cell pointer, @@ -70234,7 +70595,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( iStart<=pPage->pBt->usableSize-4 ); + assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 ); /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. @@ -70291,6 +70652,11 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ } pTmp = &data[hdr+5]; x = get2byte(pTmp); + if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[iStart], 0, iSize); + } if( iStart<=x ){ /* The new freeblock is at the beginning of the cell content area, ** so just extend the cell content area rather than create another @@ -70302,14 +70668,9 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ }else{ /* Insert the new freeblock into the freelist */ put2byte(&data[iPtr], iStart); + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); } - if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ - memset(&data[iStart], 0, iSize); - } - put2byte(&data[iStart], iFreeBlk); - put2byte(&data[iStart+2], iSize); pPage->nFree += iOrigSize; return SQLITE_OK; } @@ -70346,14 +70707,14 @@ static int decodeFlags(MemPage *pPage, int flagByte){ }else if( flagByte==(PTF_ZERODATA | PTF_LEAF) ){ pPage->intKey = 0; pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; + pPage->xCellSize = cellSizePtrIdxLeaf; pPage->xParseCell = btreeParseCellPtrIndex; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ pPage->intKey = 0; pPage->intKeyLeaf = 0; - pPage->xCellSize = cellSizePtr; + pPage->xCellSize = cellSizePtrIdxLeaf; pPage->xParseCell = btreeParseCellPtrIndex; return SQLITE_CORRUPT_PAGE(pPage); } @@ -72219,7 +72580,7 @@ static int relocatePage( if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; /* Move page iDbPage from its current location to page number iFreePage */ - TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", + TRACE(("AUTOVACUUM: Moving %u to free page %u (ptr page %u type %u)\n", iDbPage, iFreePage, iPtrPage, eType)); rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); if( rc!=SQLITE_OK ){ @@ -74505,7 +74866,8 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ pPage = pCur->pPage; idx = ++pCur->ix; - if( NEVER(!pPage->isInit) || sqlite3FaultSim(412) ){ + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ return SQLITE_CORRUPT_BKPT; } @@ -74768,7 +75130,7 @@ static int allocateBtreePage( memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); *ppPage = pTrunk; pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); }else if( k>(u32)(pBt->usableSize/4 - 2) ){ /* Value of k is out of range. Database corruption */ rc = SQLITE_CORRUPT_PGNO(iTrunk); @@ -74834,7 +75196,7 @@ static int allocateBtreePage( } } pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); #endif }else if( k>0 ){ /* Extract a leaf from the trunk */ @@ -74879,8 +75241,8 @@ static int allocateBtreePage( ){ int noContent; *pPgno = iPage; - TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d" - ": %d more free pages\n", + TRACE(("ALLOCATE: %u was leaf %u of %u on trunk %u" + ": %u more free pages\n", *pPgno, closest+1, k, pTrunk->pgno, n-1)); rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc ) goto end_allocate_page; @@ -74936,7 +75298,7 @@ static int allocateBtreePage( ** becomes a new pointer-map page, the second is used by the caller. */ MemPage *pPg = 0; - TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); + TRACE(("ALLOCATE: %u from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ @@ -74959,7 +75321,7 @@ static int allocateBtreePage( releasePage(*ppPage); *ppPage = 0; } - TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); + TRACE(("ALLOCATE: %u from end of file\n", *pPgno)); } assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) ); @@ -75087,7 +75449,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ } rc = btreeSetHasContent(pBt, iPage); } - TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); + TRACE(("FREE-PAGE: %u leaf on trunk page %u\n",pPage->pgno,pTrunk->pgno)); goto freepage_out; } } @@ -75108,7 +75470,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ put4byte(pPage->aData, iTrunk); put4byte(&pPage->aData[4], 0); put4byte(&pPage1->aData[32], iPage); - TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); + TRACE(("FREE-PAGE: %u new trunk page replacing %u\n", pPage->pgno, iTrunk)); freepage_out: if( pPage ){ @@ -75467,6 +75829,14 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. +** +** The insertCellFast() routine below works exactly the same as +** insertCell() except that it lacks the pTemp and iChild parameters +** which are assumed zero. Other than that, the two routines are the +** same. +** +** Fixes or enhancements to this routine should be reflected in +** insertCellFast()! */ static int insertCell( MemPage *pPage, /* Page into which we are copying */ @@ -75489,14 +75859,103 @@ static int insertCell( assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); assert( pPage->nFree>=0 ); + assert( iChild>0 ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); pCell = pTemp; } - if( iChild ){ - put4byte(pCell, iChild); + put4byte(pCell, iChild); + j = pPage->nOverflow++; + /* Comparison against ArraySize-1 since we hold back one extra slot + ** as a contingency. In other words, never need more than 3 overflow + ** slots but 4 are allocated, just to be safe. */ + assert( j < ArraySize(pPage->apOvfl)-1 ); + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ + }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( NEVER(rc!=SQLITE_OK) ){ + return rc; + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + data = pPage->aData; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); + rc = allocateSpace(pPage, sz, &idx); + if( rc ){ return rc; } + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); + assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nFree -= (u16)(2 + sz); + /* In a corrupt database where an entry in the cell index section of + ** a btree page has a value of 3 or less, the pCell value might point + ** as many as 4 bytes in front of the start of the aData buffer for + ** the source page. Make sure this does not cause problems by not + ** reading the first 4 bytes */ + memcpy(&data[idx+4], pCell+4, sz-4); + put4byte(&data[idx], iChild); + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pPage->pBt->autoVacuum ){ + int rc2 = SQLITE_OK; + /* The cell may contain a pointer to an overflow page. If so, write + ** the entry for the overflow page into the pointer map. + */ + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc2; } +#endif + } + return SQLITE_OK; +} + +/* +** This variant of insertCell() assumes that the pTemp and iChild +** parameters are both zero. Use this variant in sqlite3BtreeInsert() +** for performance improvement, and also so that this variant is only +** called from that one place, and is thus inlined, and thus runs must +** faster. +** +** Fixes or enhancements to this routine should be reflected into +** the insertCell() routine. +*/ +static int insertCellFast( + MemPage *pPage, /* Page into which we are copying */ + int i, /* New cell becomes the i-th cell of the page */ + u8 *pCell, /* Content of the new cell */ + int sz /* Bytes of content in pCell */ +){ + int idx = 0; /* Where to write new cell content in data[] */ + int j; /* Loop counter */ + u8 *data; /* The content of the whole page */ + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ + + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + assert( pPage->nOverflow==0 ); + if( sz+2>pPage->nFree ){ j = pPage->nOverflow++; /* Comparison against ArraySize-1 since we hold back one extra slot ** as a contingency. In other words, never need more than 3 overflow @@ -75528,17 +75987,7 @@ static int insertCell( assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); pPage->nFree -= (u16)(2 + sz); - if( iChild ){ - /* In a corrupt database where an entry in the cell index section of - ** a btree page has a value of 3 or less, the pCell value might point - ** as many as 4 bytes in front of the start of the aData buffer for - ** the source page. Make sure this does not cause problems by not - ** reading the first 4 bytes */ - memcpy(&data[idx+4], pCell+4, sz-4); - put4byte(&data[idx], iChild); - }else{ - memcpy(&data[idx], pCell, sz); - } + memcpy(&data[idx], pCell, sz); pIns = pPage->aCellIdx + i*2; memmove(pIns+2, pIns, 2*(pPage->nCell - i)); put2byte(pIns, idx); @@ -75723,7 +76172,7 @@ static int rebuildPage( assert( i(u32)usableSize ){ j = 0; } + if( NEVER(j>(u32)usableSize) ){ j = 0; } memcpy(&pTmp[j], &aData[j], usableSize - j); for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kpBt->usableSize]; u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; - int i; + int i, j; int iEnd = iFirst + nCell; - u8 *pFree = 0; /* \__ Parameters for pending call to */ - int szFree = 0; /* / freeSpace() */ + int nFree = 0; + int aOfst[10]; + int aAfter[10]; for(i=iFirst; iapCell[i]; if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ int sz; + int iAfter; + int iOfst; /* No need to use cachedCellSize() here. The sizes of all cells that ** are to be freed have already been computing while deciding which ** cells need freeing */ sz = pCArray->szCell[i]; assert( sz>0 ); - if( pFree!=(pCell + sz) ){ - if( pFree ){ - assert( pFree>aData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); - } - pFree = pCell; - szFree = sz; - if( pFree+sz>pEnd ){ - return 0; + iOfst = (u16)(pCell - aData); + iAfter = iOfst+sz; + for(j=0; j=nFree ){ + if( nFree>=(int)(sizeof(aOfst)/sizeof(aOfst[0])) ){ + for(j=0; jpEnd ) return 0; + nFree++; } nRet++; } } - if( pFree ){ - assert( pFree>aData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); + for(j=0; jpPg->aDataEnd ) goto editpage_fail; + if( NEVER(pData>pPg->aDataEnd) ) goto editpage_fail; /* Add cells to the start of the page */ if( iNew0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); - TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + TRACE(("BALANCE: old: %u(nc=%u) %u(nc=%u) %u(nc=%u)\n", apOld[0]->pgno, apOld[0]->nCell, nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 @@ -76778,8 +77235,8 @@ static int balance_nonroot( } } - TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " - "%d(%d nc=%d) %d(%d nc=%d)\n", + TRACE(("BALANCE: new: %u(%u nc=%u) %u(%u nc=%u) %u(%u nc=%u) " + "%u(%u nc=%u) %u(%u nc=%u)\n", apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, @@ -77024,7 +77481,7 @@ static int balance_nonroot( } assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + TRACE(("BALANCE: finished: old=%u new=%u cells=%u\n", nOld, nNew, b.nCell)); /* Free any old pages that were not reused as new pages. @@ -77109,7 +77566,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); - TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); + TRACE(("BALANCE: copy root %u into %u\n", pRoot->pgno, pChild->pgno)); /* Copy the overflow cells from pRoot to pChild */ memcpy(pChild->aiOvfl, pRoot->aiOvfl, @@ -77592,7 +78049,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( } } assert( pCur->eState==CURSOR_VALID - || (pCur->eState==CURSOR_INVALID && loc) ); + || (pCur->eState==CURSOR_INVALID && loc) || CORRUPT_DB ); pPage = pCur->pPage; assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) ); @@ -77607,7 +78064,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( rc ) return rc; } - TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", + TRACE(("INSERT: table=%u nkey=%lld ndata=%u page=%u %s\n", pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit || CORRUPT_DB ); @@ -77634,6 +78091,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( assert( szNew==pPage->xCellSize(pPage, newCell) ); assert( szNew <= MX_CELL_SIZE(p->pBt) ); idx = pCur->ix; + pCur->info.nSize = 0; if( loc==0 ){ CellInfo info; assert( idx>=0 ); @@ -77682,7 +78140,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( }else{ assert( pPage->leaf ); } - rc = insertCell(pPage, idx, newCell, szNew, 0, 0); + rc = insertCellFast(pPage, idx, newCell, szNew); assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); @@ -77706,7 +78164,6 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** larger than the largest existing key, it is possible to insert the ** row without seeking the cursor. This can be a big performance boost. */ - pCur->info.nSize = 0; if( pPage->nOverflow ){ assert( rc==SQLITE_OK ); pCur->curFlags &= ~(BTCF_ValidNKey); @@ -77907,6 +78364,9 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ return SQLITE_CORRUPT_BKPT; } + if( pCell<&pPage->aCellIdx[pPage->nCell] ){ + return SQLITE_CORRUPT_BKPT; + } /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must ** be preserved following this delete operation. If the current delete @@ -78655,7 +79115,8 @@ static void checkAppendMsg( sqlite3_str_append(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ - sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, + pCheck->v0, pCheck->v1, pCheck->v2); } sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap); va_end(ap); @@ -78695,11 +79156,11 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ */ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ if( iPage>pCheck->nPage || iPage==0 ){ - checkAppendMsg(pCheck, "invalid page number %d", iPage); + checkAppendMsg(pCheck, "invalid page number %u", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ - checkAppendMsg(pCheck, "2nd reference to page %d", iPage); + checkAppendMsg(pCheck, "2nd reference to page %u", iPage); return 1; } setPageReferenced(pCheck, iPage); @@ -78725,13 +79186,13 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) checkOom(pCheck); - checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); + checkAppendMsg(pCheck, "Failed to read ptrmap key=%u", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ checkAppendMsg(pCheck, - "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", + "Bad ptr map entry key=%u expected=(%u,%u) got=(%u,%u)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } } @@ -78756,7 +79217,7 @@ static void checkList( if( checkRef(pCheck, iPage) ) break; N--; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ - checkAppendMsg(pCheck, "failed to get page %d", iPage); + checkAppendMsg(pCheck, "failed to get page %u", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); @@ -78769,7 +79230,7 @@ static void checkList( #endif if( n>pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, - "freelist leaf count too big on page %d", iPage); + "freelist leaf count too big on page %u", iPage); N--; }else{ for(i=0; i<(int)n; i++){ @@ -78801,7 +79262,7 @@ static void checkList( } if( N && nErrAtStart==pCheck->nErr ){ checkAppendMsg(pCheck, - "%s is %d but should be %d", + "%s is %u but should be %u", isFreeList ? "size" : "overflow list length", expected-N, expected); } @@ -78916,8 +79377,8 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; - pCheck->zPfx = "Page %u: "; - pCheck->v1 = iPage; + pCheck->zPfx = "Tree %u page %u: "; + pCheck->v0 = pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); @@ -78943,7 +79404,7 @@ static int checkTreePage( hdr = pPage->hdrOffset; /* Set up for cell analysis */ - pCheck->zPfx = "On tree page %u cell %d: "; + pCheck->zPfx = "Tree %u page %u cell %u: "; contentOffset = get2byteNotZero(&data[hdr+5]); assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ @@ -78963,7 +79424,7 @@ static int checkTreePage( pgno = get4byte(&data[hdr+8]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - pCheck->zPfx = "On page %u at right child: "; + pCheck->zPfx = "Tree %u page %u right child: "; checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif @@ -78987,7 +79448,7 @@ static int checkTreePage( pc = get2byteAligned(pCellIdx); pCellIdx -= 2; if( pcusableSize-4 ){ - checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + checkAppendMsg(pCheck, "Offset %u out of range %u..%u", pc, contentOffset, usableSize-4); doCoverageCheck = 0; continue; @@ -79119,7 +79580,7 @@ static int checkTreePage( */ if( heap[0]==0 && nFrag!=data[hdr+7] ){ checkAppendMsg(pCheck, - "Fragmentation of %d bytes reported as %d on page %u", + "Fragmentation of %u bytes reported as %u on page %u", nFrag, data[hdr+7], iPage); } } @@ -79216,7 +79677,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( /* Check the integrity of the freelist */ if( bCkFreelist ){ - sCheck.zPfx = "Main freelist: "; + sCheck.zPfx = "Freelist: "; checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), get4byte(&pBt->pPage1->aData[36])); sCheck.zPfx = 0; @@ -79233,7 +79694,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( mxInHdr = get4byte(&pBt->pPage1->aData[52]); if( mx!=mxInHdr ){ checkAppendMsg(&sCheck, - "max rootpage (%d) disagrees with header (%d)", + "max rootpage (%u) disagrees with header (%u)", mx, mxInHdr ); } @@ -79264,7 +79725,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %u: never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain @@ -79272,11 +79733,11 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( */ if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %u: never used", i); } if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); + checkAppendMsg(&sCheck, "Page %u: pointer map referenced", i); } #endif } @@ -79838,13 +80299,7 @@ static int backupOnePage( assert( !isFatalError(p->rc) ); assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); assert( zSrcData ); - - /* Catch the case where the destination is an in-memory database and the - ** page sizes of the source and destination differ. - */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ - rc = SQLITE_READONLY; - } + assert( nSrcPgsz==nDestPgsz || sqlite3PagerIsMemdb(pDestPager)==0 ); /* This loop runs once for each destination page spanned by the source ** page. For each iteration, variable iOff is set to the byte offset @@ -79977,7 +80432,10 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); pgszDest = sqlite3BtreeGetPageSize(p->pDest); destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); - if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ + if( SQLITE_OK==rc + && (destMode==PAGER_JOURNALMODE_WAL || sqlite3PagerIsMemdb(pDestPager)) + && pgszSrc!=pgszDest + ){ rc = SQLITE_READONLY; } @@ -80526,6 +80984,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ char *z; int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; + if( p->db && p->db->mallocFailed ) return 1; if( p->flags & MEM_Term ){ /* Insure that the string is properly zero-terminated. Pay particular ** attention to the case where p->n is odd */ @@ -80808,7 +81267,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ vdbeMemRenderNum(nByte, pMem->z, pMem); assert( pMem->z!=0 ); - assert( pMem->n==sqlite3Strlen30NN(pMem->z) ); + assert( pMem->n==(int)sqlite3Strlen30NN(pMem->z) ); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); @@ -81852,6 +82311,9 @@ static int valueFromFunction( if( pList ) nVal = pList->nExpr; assert( !ExprHasProperty(p, EP_IntValue) ); pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pFunc==0 ) return SQLITE_OK; +#endif assert( pFunc ); if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) @@ -81888,16 +82350,11 @@ static int valueFromFunction( }else{ sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); assert( rc==SQLITE_OK ); - assert( enc==pVal->enc - || (pVal->flags & MEM_Str)==0 - || db->mallocFailed ); -#if 0 /* Not reachable except after a prior failure */ rc = sqlite3VdbeChangeEncoding(pVal, enc); - if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){ + if( NEVER(rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal)) ){ rc = SQLITE_TOOBIG; pCtx->pParse->nErr++; } -#endif } value_from_function_out: @@ -81961,6 +82418,13 @@ static int valueFromExpr( rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); testcase( rc!=SQLITE_OK ); if( *ppVal ){ +#ifdef SQLITE_ENABLE_STAT4 + rc = ExpandBlob(*ppVal); +#else + /* zero-blobs only come from functions, not literal values. And + ** functions are only processed under STAT4 */ + assert( (ppVal[0][0].flags & MEM_Zero)==0 ); +#endif sqlite3VdbeMemCast(*ppVal, aff, enc); sqlite3ValueApplyAffinity(*ppVal, affinity, enc); } @@ -82807,10 +83271,10 @@ SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ */ SQLITE_PRIVATE int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ int addr = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) +#if !defined(SQLITE_DEBUG) /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. ** But omit them (for performance) during production builds */ - if( pParse->explain==2 ) + if( pParse->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { char *zMsg; @@ -83184,6 +83648,8 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; + + assert( pParse->db->mallocFailed==0 ); /* tag-20230419-1 */ p->readOnly = 1; p->bIsReader = 0; pOp = &p->aOp[p->nOp-1]; @@ -83243,6 +83709,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); assert( ADDR(pOp->p2)<-pParse->nLabel ); + assert( aLabel!=0 ); /* True because of tag-20230419-1 */ pOp->p2 = aLabel[ADDR(pOp->p2)]; } break; @@ -83486,18 +83953,20 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); - ScanStatus *aNew; - aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); - if( aNew ){ - ScanStatus *pNew = &aNew[p->nScan++]; - memset(pNew, 0, sizeof(ScanStatus)); - pNew->addrExplain = addrExplain; - pNew->addrLoop = addrLoop; - pNew->addrVisit = addrVisit; - pNew->nEst = nEst; - pNew->zName = sqlite3DbStrDup(p->db, zName); - p->aScan = aNew; + if( IS_STMT_SCANSTATUS(p->db) ){ + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + memset(pNew, 0, sizeof(ScanStatus)); + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } } } @@ -83514,20 +83983,22 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatusRange( int addrStart, int addrEnd ){ - ScanStatus *pScan = 0; - int ii; - for(ii=p->nScan-1; ii>=0; ii--){ - pScan = &p->aScan[ii]; - if( pScan->addrExplain==addrExplain ) break; - pScan = 0; - } - if( pScan ){ - if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; - for(ii=0; iiaAddrRange); ii+=2){ - if( pScan->aAddrRange[ii]==0 ){ - pScan->aAddrRange[ii] = addrStart; - pScan->aAddrRange[ii+1] = addrEnd; - break; + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; + for(ii=0; iiaAddrRange); ii+=2){ + if( pScan->aAddrRange[ii]==0 ){ + pScan->aAddrRange[ii] = addrStart; + pScan->aAddrRange[ii+1] = addrEnd; + break; + } } } } @@ -83544,19 +84015,21 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters( int addrLoop, int addrVisit ){ - ScanStatus *pScan = 0; - int ii; - for(ii=p->nScan-1; ii>=0; ii--){ - pScan = &p->aScan[ii]; - if( pScan->addrExplain==addrExplain ) break; - pScan = 0; - } - if( pScan ){ - pScan->addrLoop = addrLoop; - pScan->addrVisit = addrVisit; + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + pScan->addrLoop = addrLoop; + pScan->addrVisit = addrVisit; + } } } -#endif +#endif /* defined(SQLITE_ENABLE_STMT_SCANSTATUS) */ /* @@ -83980,7 +84453,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ /* Return the most recently added opcode */ -VdbeOp * sqlite3VdbeGetLastOp(Vdbe *p){ +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe *p){ return sqlite3VdbeGetOp(p, p->nOp - 1); } @@ -85684,6 +86157,8 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } + }else if( p->rc==SQLITE_SCHEMA && db->nVdbeActive>1 ){ + p->nChange = 0; }else{ sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; @@ -86002,9 +86477,9 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ #ifdef SQLITE_ENABLE_NORMALIZE sqlite3DbFree(db, p->zNormSql); { - DblquoteStr *pThis, *pNext; - for(pThis=p->pDblStr; pThis; pThis=pNext){ - pNext = pThis->pNextStr; + DblquoteStr *pThis, *pNxt; + for(pThis=p->pDblStr; pThis; pThis=pNxt){ + pNxt = pThis->pNextStr; sqlite3DbFree(db, pThis); } } @@ -87631,6 +88106,20 @@ SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ return 1; } +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +/* +** This Walker callback is used to help verify that calls to +** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have +** byte-code register values correctly initialized. +*/ +SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_REGISTER ){ + assert( (pWalker->u.aMem[pExpr->iTable].flags & MEM_Undefined)==0 ); + } + return WRC_Continue; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS && SQLITE_DEBUG */ + #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored @@ -87693,6 +88182,16 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( PreUpdate preupdate; const char *zTbl = pTab->zName; static const u8 fakeSortOrder = 0; +#ifdef SQLITE_DEBUG + int nRealCol; + if( pTab->tabFlags & TF_WithoutRowid ){ + nRealCol = sqlite3PrimaryKeyIndex(pTab)->nColumn; + }else if( pTab->tabFlags & TF_HasVirtual ){ + nRealCol = pTab->nNVCol; + }else{ + nRealCol = pTab->nCol; + } +#endif assert( db->pPreUpdate==0 ); memset(&preupdate, 0, sizeof(PreUpdate)); @@ -87709,8 +88208,8 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( assert( pCsr!=0 ); assert( pCsr->eCurType==CURTYPE_BTREE ); - assert( pCsr->nField==pTab->nCol - || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1) + assert( pCsr->nField==nRealCol + || (pCsr->nField==nRealCol+1 && op==SQLITE_DELETE && iReg==-1) ); preupdate.v = v; @@ -88017,7 +88516,7 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ SQLITE_NULL, /* 0x1f (not possible) */ SQLITE_FLOAT, /* 0x20 INTREAL */ SQLITE_NULL, /* 0x21 (not possible) */ - SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ + SQLITE_FLOAT, /* 0x22 INTREAL + TEXT */ SQLITE_NULL, /* 0x23 (not possible) */ SQLITE_FLOAT, /* 0x24 (not possible) */ SQLITE_NULL, /* 0x25 (not possible) */ @@ -89083,9 +89582,9 @@ static const void *columnName( assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N=0 ){ + u8 prior_mallocFailed = db->mallocFailed; N += useType*n; sqlite3_mutex_enter(db->mutex); - assert( db->mallocFailed==0 ); #ifndef SQLITE_OMIT_UTF16 if( useUtf16 ){ ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); @@ -89097,7 +89596,8 @@ static const void *columnName( /* A malloc may have failed inside of the _text() call. If this ** is the case, clear the mallocFailed flag and return NULL. */ - if( db->mallocFailed ){ + assert( db->mallocFailed==0 || db->mallocFailed==1 ); + if( db->mallocFailed > prior_mallocFailed ){ sqlite3OomClear(db); ret = 0; } @@ -89884,15 +90384,24 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; - ScanStatus *pScan; + VdbeOp *aOp = p->aOp; + int nOp = p->nOp; + ScanStatus *pScan = 0; int idx; + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + aOp = pFrame->aOp; + nOp = pFrame->nOp; + } + if( iScan<0 ){ int ii; if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ i64 res = 0; - for(ii=0; iinOp; ii++){ - res += p->aOp[ii].nCycle; + for(ii=0; iiaddrLoop>0 ){ - *(sqlite3_int64*)pOut = p->aOp[pScan->addrLoop].nExec; + *(sqlite3_int64*)pOut = aOp[pScan->addrLoop].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -89926,7 +90435,7 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_NVISIT: { if( pScan->addrVisit>0 ){ - *(sqlite3_int64*)pOut = p->aOp[pScan->addrVisit].nExec; + *(sqlite3_int64*)pOut = aOp[pScan->addrVisit].nExec; }else{ *(sqlite3_int64*)pOut = -1; } @@ -89948,7 +90457,7 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_EXPLAIN: { if( pScan->addrExplain ){ - *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + *(const char**)pOut = aOp[ pScan->addrExplain ].p4.z; }else{ *(const char**)pOut = 0; } @@ -89956,7 +90465,7 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_SELECTID: { if( pScan->addrExplain ){ - *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + *(int*)pOut = aOp[ pScan->addrExplain ].p1; }else{ *(int*)pOut = -1; } @@ -89964,7 +90473,7 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( } case SQLITE_SCANSTAT_PARENTID: { if( pScan->addrExplain ){ - *(int*)pOut = p->aOp[ pScan->addrExplain ].p2; + *(int*)pOut = aOp[ pScan->addrExplain ].p2; }else{ *(int*)pOut = -1; } @@ -89982,18 +90491,18 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2( if( iIns==0 ) break; if( iIns>0 ){ while( iIns<=iEnd ){ - res += p->aOp[iIns].nCycle; + res += aOp[iIns].nCycle; iIns++; } }else{ int iOp; - for(iOp=0; iOpnOp; iOp++){ - Op *pOp = &p->aOp[iOp]; + for(iOp=0; iOpp1!=iEnd ) continue; if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ continue; } - res += p->aOp[iOp].nCycle; + res += aOp[iOp].nCycle; } } } @@ -90916,7 +91425,10 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){ }else if( p->flags & MEM_Real ){ h += sqlite3VdbeIntValue(p); }else if( p->flags & (MEM_Str|MEM_Blob) ){ - /* no-op */ + /* All strings have the same hash and all blobs have the same hash, + ** though, at least, those hashes are different from each other and + ** from NULL. */ + h += 4093 + (p->flags & (MEM_Str|MEM_Blob)); } } return h; @@ -90966,6 +91478,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( Mem *pOut = 0; /* Output operand */ #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) u64 *pnCycle = 0; + int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0; #endif /*** INSERT STACK UNION HERE ***/ @@ -91030,13 +91543,17 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( pOp>=aOp && pOp<&aOp[p->nOp]); nVmStep++; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + +#if defined(VDBE_PROFILE) pOp->nExec++; pnCycle = &pOp->nCycle; -# ifdef VDBE_PROFILE - if( sqlite3NProfileCnt==0 ) -# endif + if( sqlite3NProfileCnt==0 ) *pnCycle -= sqlite3Hwtime(); +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( bStmtScanStatus ){ + pOp->nExec++; + pnCycle = &pOp->nCycle; *pnCycle -= sqlite3Hwtime(); + } #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -92624,7 +93141,7 @@ case OP_Compare: { /* Opcode: Jump P1 P2 P3 * * ** ** Jump to the instruction at address P1, P2, or P3 depending on whether -** in the most recent OP_Compare instruction the P1 vector was less than +** in the most recent OP_Compare instruction the P1 vector was less than, ** equal to, or greater than the P2 vector, respectively. ** ** This opcode must immediately follow an OP_Compare opcode. @@ -92851,6 +93368,12 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ ** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04. ** SQLITE_BLOB is 0x08. SQLITE_NULL is 0x10. ** +** WARNING: This opcode does not reliably distinguish between NULL and REAL +** when P1>=0. If the database contains a NaN value, this opcode will think +** that the datatype is REAL when it should be NULL. When P1<0 and the value +** is already stored in register P3, then this opcode does reliably +** distinguish between NULL and REAL. The problem only arises then P1>=0. +** ** Take the jump to address P2 if and only if the datatype of the ** value determined by P1 and P3 corresponds to one of the bits in the ** P5 bitmask. @@ -92964,7 +93487,7 @@ case OP_IfNullRow: { /* jump */ VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - if( ALWAYS(pC) && pC->nullRow ){ + if( pC && pC->nullRow ){ sqlite3VdbeMemSetNull(aMem + pOp->p3); goto jump_to_p2; } @@ -93459,7 +93982,7 @@ case OP_Affinity: { }else{ pIn1->u.r = (double)pIn1->u.i; pIn1->flags |= MEM_Real; - pIn1->flags &= ~MEM_Int; + pIn1->flags &= ~(MEM_Int|MEM_Str); } } REGISTER_TRACE((int)(pIn1-aMem), pIn1); @@ -95198,6 +95721,7 @@ case OP_SeekScan: { /* ncycle */ break; } nStep--; + pC->cacheStatus = CACHE_STALE; rc = sqlite3BtreeNext(pC->uc.pCursor, 0); if( rc ){ if( rc==SQLITE_DONE ){ @@ -97850,6 +98374,7 @@ case OP_AggFinal: { } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); + REGISTER_TRACE((int)(pMem-aMem), pMem); break; } @@ -98988,8 +99513,10 @@ default: { /* This is really OP_Noop, OP_Explain */ *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); pnCycle = 0; #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) - *pnCycle += sqlite3Hwtime(); - pnCycle = 0; + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; + } #endif /* The following code adds nothing to the actual functionality @@ -99468,7 +99995,7 @@ SQLITE_API int sqlite3_blob_open( if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); sqlite3DbFree(db, pBlob); } - sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr); sqlite3DbFree(db, zErr); sqlite3ParseObjectReset(&sParse); rc = sqlite3ApiExit(db, rc); @@ -99627,7 +100154,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ ((Vdbe*)p->pStmt)->rc = SQLITE_OK; rc = blobSeekToRow(p, iRow, &zErr); if( rc!=SQLITE_OK ){ - sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr); sqlite3DbFree(db, zErr); } assert( rc!=SQLITE_SCHEMA ); @@ -104015,7 +104542,8 @@ static int lookupName( assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); if( pParse->bReturning ){ if( (pNC->ncFlags & NC_UBaseReg)!=0 - && (zTab==0 || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + && ALWAYS(zTab==0 + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) ){ pExpr->iTable = op!=TK_DELETE; pTab = pParse->pTriggerTab; @@ -104800,8 +105328,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); - pNC->ncFlags |= NC_VarSelect; } + pNC->ncFlags |= NC_Subquery; } break; } @@ -105989,11 +106517,10 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ }else{ Expr *pNext = p->pRight; /* The Expr.x union is never used at the same time as Expr.pRight */ - assert( ExprUseXList(p) ); - assert( p->x.pList==0 || p->pRight==0 ); - if( p->x.pList!=0 && !db->mallocFailed ){ + assert( !ExprUseXList(p) || p->x.pList==0 || p->pRight==0 ); + if( ExprUseXList(p) && p->x.pList!=0 && !db->mallocFailed ){ int i; - for(i=0; ALWAYS(ix.pList->nExpr); i++){ + for(i=0; ix.pList->nExpr; i++){ if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ pNext = p->x.pList->a[i].pExpr; break; @@ -106825,9 +107352,9 @@ SQLITE_PRIVATE Select *sqlite3ExprListToValues(Parse *pParse, int nElem, ExprLis ** Join two expressions using an AND operator. If either expression is ** NULL, then just return the other expression. ** -** If one side or the other of the AND is known to be false, then instead -** of returning an AND expression, just return a constant expression with -** a value of false. +** If one side or the other of the AND is known to be false, and neither side +** is part of an ON clause, then instead of returning an AND expression, +** just return a constant expression with a value of false. */ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ sqlite3 *db = pParse->db; @@ -106835,14 +107362,17 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ return pRight; }else if( pRight==0 ){ return pLeft; - }else if( (ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight)) - && !IN_RENAME_OBJECT - ){ - sqlite3ExprDeferredDelete(pParse, pLeft); - sqlite3ExprDeferredDelete(pParse, pRight); - return sqlite3Expr(db, TK_INTEGER, "0"); }else{ - return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + u32 f = pLeft->flags | pRight->flags; + if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse + && !IN_RENAME_OBJECT + ){ + sqlite3ExprDeferredDelete(pParse, pLeft); + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3Expr(db, TK_INTEGER, "0"); + }else{ + return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + } } } @@ -108087,12 +108617,17 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ } /* -** Check pExpr to see if it is an invariant constraint on data source pSrc. +** Check pExpr to see if it is an constraint on the single data source +** pSrc = &pSrcList->a[iSrc]. In other words, check to see if pExpr +** constrains pSrc but does not depend on any other tables or data +** sources anywhere else in the query. Return true (non-zero) if pExpr +** is a constraint on pSrc only. +** ** This is an optimization. False negatives will perhaps cause slower ** queries, but false positives will yield incorrect answers. So when in ** doubt, return 0. ** -** To be an invariant constraint, the following must be true: +** To be an single-source constraint, the following must be true: ** ** (1) pExpr cannot refer to any table other than pSrc->iCursor. ** @@ -108103,13 +108638,31 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ ** ** (4) If pSrc is the right operand of a LEFT JOIN, then... ** (4a) pExpr must come from an ON clause.. - (4b) and specifically the ON clause associated with the LEFT JOIN. +** (4b) and specifically the ON clause associated with the LEFT JOIN. ** ** (5) If pSrc is not the right operand of a LEFT JOIN or the left ** operand of a RIGHT JOIN, then pExpr must be from the WHERE ** clause, not an ON clause. +** +** (6) Either: +** +** (6a) pExpr does not originate in an ON or USING clause, or +** +** (6b) The ON or USING clause from which pExpr is derived is +** not to the left of a RIGHT JOIN (or FULL JOIN). +** +** Without this restriction, accepting pExpr as a single-table +** constraint might move the the ON/USING filter expression +** from the left side of a RIGHT JOIN over to the right side, +** which leads to incorrect answers. See also restriction (9) +** on push-down. */ -SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr *pExpr, const SrcItem *pSrc){ +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( + Expr *pExpr, /* The constraint */ + const SrcList *pSrcList, /* Complete FROM clause */ + int iSrc /* Which element of pSrcList to use */ +){ + const SrcItem *pSrc = &pSrcList->a[iSrc]; if( pSrc->fg.jointype & JT_LTORJ ){ return 0; /* rule (3) */ } @@ -108119,6 +108672,19 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr *pExpr, const SrcItem *pSrc }else{ if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* rule (5) */ } + if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON) /* (6a) */ + && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0 /* Fast pre-test of (6b) */ + ){ + int jj; + for(jj=0; jjw.iJoin==pSrcList->a[jj].iCursor ){ + if( (pSrcList->a[jj].fg.jointype & JT_LTORJ)!=0 ){ + return 0; /* restriction (6) */ + } + break; + } + } + } return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */ } @@ -108361,7 +108927,7 @@ SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ ** pX is the RHS of an IN operator. If pX is a SELECT statement ** that can be simplified to a direct table access, then return ** a pointer to the SELECT statement. If pX is not a SELECT statement, -** or if the SELECT statement needs to be manifested into a transient +** or if the SELECT statement needs to be materialized into a transient ** table, then return NULL. */ #ifndef SQLITE_OMIT_SUBQUERY @@ -108647,7 +109213,6 @@ SQLITE_PRIVATE int sqlite3FindInIndex( CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); int j; - assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr ); for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; assert( pIdx->azColl[j] ); @@ -109550,6 +110115,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( ){ int iAddr; Vdbe *v = pParse->pVdbe; + int nErr = pParse->nErr; assert( v!=0 ); assert( pParse->iSelfTab!=0 ); if( pParse->iSelfTab>0 ){ @@ -109562,6 +110128,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); } if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); + if( pParse->nErr>nErr ) pParse->db->errByteOffset = -1; } #endif /* SQLITE_OMIT_GENERATED_COLUMNS */ @@ -109578,6 +110145,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( Column *pCol; assert( v!=0 ); assert( pTab!=0 ); + assert( iCol!=XN_EXPR ); if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); VdbeComment((v, "%s.rowid", pTab->zName)); @@ -109930,7 +110498,19 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) AggInfo *pAggInfo = pExpr->pAggInfo; struct AggInfo_col *pCol; assert( pAggInfo!=0 ); - assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + assert( pExpr->iAgg>=0 ); + if( pExpr->iAgg>=pAggInfo->nColumn ){ + /* Happens when the left table of a RIGHT JOIN is null and + ** is using an expression index */ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); +#ifdef SQLITE_VDBE_COVERAGE + /* Verify that the OP_Null above is exercised by tests + ** tag-20230325-2 */ + sqlite3VdbeAddOp2(v, OP_NotNull, target, 1); + VdbeCoverageNeverTaken(v); +#endif + break; + } pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ return AggInfoColumnReg(pAggInfo, pExpr->iAgg); @@ -110105,11 +110685,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - if( inReg!=target ){ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( inReg==target ); assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3VdbeAddOp2(v, OP_Cast, target, sqlite3AffinityType(pExpr->u.zToken, 0)); @@ -110448,13 +111025,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** Clear subtypes as subtypes may not cross a subquery boundary. */ assert( pExpr->pLeft ); - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - if( inReg!=target ){ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } - sqlite3VdbeAddOp1(v, OP_ClrSubtype, inReg); - return inReg; + sqlite3ExprCode(pParse, pExpr->pLeft, target); + sqlite3VdbeAddOp1(v, OP_ClrSubtype, target); + return target; }else{ pExpr = pExpr->pLeft; goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. */ @@ -110564,12 +111137,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** "target" and not someplace else. */ pParse->okConstFactor = 0; /* note (1) above */ - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( target==inReg ); pParse->okConstFactor = okConstFactor; - if( inReg!=target ){ /* note (2) above */ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } sqlite3VdbeJumpHere(v, addrINR); break; } @@ -110807,7 +111377,9 @@ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); if( inReg!=target ){ u8 op; - if( ALWAYS(pExpr) && ExprHasProperty(pExpr,EP_Subquery) ){ + if( ALWAYS(pExpr) + && (ExprHasProperty(pExpr,EP_Subquery) || pExpr->op==TK_REGISTER) + ){ op = OP_Copy; }else{ op = OP_SCopy; @@ -111992,9 +112564,11 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ int iAgg = pExpr->iAgg; Parse *pParse = pWalker->pParse; sqlite3 *db = pParse->db; + assert( iAgg>=0 ); if( pExpr->op!=TK_AGG_FUNCTION ){ - assert( iAgg>=0 && iAggnColumn ); - if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){ + if( iAggnColumn + && pAggInfo->aCol[iAgg].pCExpr==pExpr + ){ pExpr = sqlite3ExprDup(db, pExpr, 0); if( pExpr ){ pAggInfo->aCol[iAgg].pCExpr = pExpr; @@ -112003,8 +112577,9 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ } }else{ assert( pExpr->op==TK_AGG_FUNCTION ); - assert( iAgg>=0 && iAggnFunc ); - if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ + if( ALWAYS(iAggnFunc) + && pAggInfo->aFunc[iAgg].pFExpr==pExpr + ){ pExpr = sqlite3ExprDup(db, pExpr, 0); if( pExpr ){ pAggInfo->aFunc[iAgg].pFExpr = pExpr; @@ -112154,7 +112729,12 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ } if( pIEpr==0 ) break; if( NEVER(!ExprUseYTab(pExpr)) ) break; - if( pExpr->pAggInfo!=0 ) break; /* Already resolved by outer context */ + for(i=0; inSrc; i++){ + if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break; + } + if( i>=pSrcList->nSrc ) break; + if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ + if( pParse->nErr ){ return WRC_Abort; } /* If we reach this point, it means that expression pExpr can be ** translated into a reference to an index column as described by @@ -112165,6 +112745,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ tmp.iTable = pIEpr->iIdxCur; tmp.iColumn = pIEpr->iIdxCol; findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + if( pParse->nErr ){ return WRC_Abort; } + assert( pAggInfo->aCol!=0 ); + assert( tmp.iAggnColumn ); pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; pExpr->pAggInfo = pAggInfo; pExpr->iAgg = tmp.iAgg; @@ -112188,7 +112771,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ } - return WRC_Prune; + return WRC_Continue; } case TK_AGG_FUNCTION: { if( (pNC->ncFlags & NC_InAggFunc)==0 @@ -112341,6 +112924,37 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nRangeReg = 0; } +/* +** Make sure sufficient registers have been allocated so that +** iReg is a valid register number. +*/ +SQLITE_PRIVATE void sqlite3TouchRegister(Parse *pParse, int iReg){ + if( pParse->nMemnMem = iReg; +} + +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) +/* +** Return the latest reusable register in the set of all registers. +** The value returned is no less than iMin. If any register iMin or +** greater is in permanent use, then return one more than that last +** permanent register. +*/ +SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse *pParse, int iMin){ + const ExprList *pList = pParse->pConstExpr; + if( pList ){ + int i; + for(i=0; inExpr; i++){ + if( pList->a[i].u.iConstExprReg>=iMin ){ + iMin = pList->a[i].u.iConstExprReg + 1; + } + } + } + pParse->nTempReg = 0; + pParse->nRangeReg = 0; + return iMin; +} +#endif /* SQLITE_ENABLE_STAT4 || SQLITE_DEBUG */ + /* ** Validate that no temporary register falls within the range of ** iFirst..iLast, inclusive. This routine is only call from within assert() @@ -112360,6 +112974,14 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ return 0; } } + if( pParse->pConstExpr ){ + ExprList *pList = pParse->pConstExpr; + for(i=0; inExpr; i++){ + int iReg = pList->a[i].u.iConstExprReg; + if( iReg==0 ) continue; + if( iReg>=iFirst && iReg<=iLast ) return 0; + } + } return 1; } #endif /* SQLITE_DEBUG */ @@ -113647,6 +114269,19 @@ static int renameEditSql( return rc; } +/* +** Set all pEList->a[].fg.eEName fields in the expression-list to val. +*/ +static void renameSetENames(ExprList *pEList, int val){ + if( pEList ){ + int i; + for(i=0; inExpr; i++){ + assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME ); + pEList->a[i].fg.eEName = val; + } + } +} + /* ** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming ** it was read from the schema of database zDb. Return SQLITE_OK if @@ -113694,7 +114329,17 @@ static int renameResolveTrigger(Parse *pParse){ pSrc = 0; rc = SQLITE_NOMEM; }else{ + /* pStep->pExprList contains an expression-list used for an UPDATE + ** statement. So the a[].zEName values are the RHS of the + ** " = " clauses of the UPDATE statement. So, before + ** running SelectPrep(), change all the eEName values in + ** pStep->pExprList to ENAME_SPAN (from their current value of + ** ENAME_NAME). This is to prevent any ids in ON() clauses that are + ** part of pSrc from being incorrectly resolved against the + ** a[].zEName values as if they were column aliases. */ + renameSetENames(pStep->pExprList, ENAME_SPAN); sqlite3SelectPrep(pParse, pSel, 0); + renameSetENames(pStep->pExprList, ENAME_NAME); rc = pParse->nErr ? SQLITE_ERROR : SQLITE_OK; assert( pStep->pExprList==0 || pStep->pExprList==pSel->pEList ); assert( pSrc==pSel->pSrc ); @@ -115643,11 +116288,15 @@ static void analyzeOneTable( int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ int regPrev = iMem; /* MUST BE LAST (see below) */ +#ifdef SQLITE_ENABLE_STAT4 + int doOnce = 1; /* Flag for a one-time computation */ +#endif #ifdef SQLITE_ENABLE_PREUPDATE_HOOK Table *pStat1 = 0; #endif - pParse->nMem = MAX(pParse->nMem, iMem); + sqlite3TouchRegister(pParse, iMem); + assert( sqlite3NoTempsInRange(pParse, regNewRowid, iMem) ); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; @@ -115753,7 +116402,7 @@ static void analyzeOneTable( ** the regPrev array and a trailing rowid (the rowid slot is required ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); + sqlite3TouchRegister(pParse, regPrev+nColTest); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -115925,7 +116574,35 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - pParse->nMem = MAX(pParse->nMem, regCol+nCol); + if( doOnce ){ + int mxCol = nCol; + Index *pX; + + /* Compute the maximum number of columns in any index */ + for(pX=pTab->pIndex; pX; pX=pX->pNext){ + int nColX; /* Number of columns in pX */ + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pX) ){ + nColX = pX->nKeyCol; + }else{ + nColX = pX->nColumn; + } + if( nColX>mxCol ) mxCol = nColX; + } + + /* Allocate space to compute results for the largest index */ + sqlite3TouchRegister(pParse, regCol+mxCol); + doOnce = 0; +#ifdef SQLITE_DEBUG + /* Verify that the call to sqlite3ClearTempRegCache() below + ** really is needed. + ** https://sqlite.org/forum/forumpost/83cb4a95a0 (2023-03-25) + */ + testcase( !sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) ); +#endif + sqlite3ClearTempRegCache(pParse); /* tag-20230325-1 */ + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) ); + } + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+nCol) ); addrNext = sqlite3VdbeCurrentAddr(v); callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); @@ -116006,6 +116683,11 @@ static void analyzeDatabase(Parse *pParse, int iDb){ for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); +#ifdef SQLITE_ENABLE_STAT4 + iMem = sqlite3FirstAvailableRegister(pParse, iMem); +#else + assert( iMem==sqlite3FirstAvailableRegister(pParse,iMem) ); +#endif } loadAnalysis(pParse, iDb); } @@ -116393,6 +117075,10 @@ static int loadStatTbl( pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); assert( pIdx==0 || pIdx->nSample==0 ); if( pIdx==0 ) continue; + if( pIdx->aSample!=0 ){ + /* The same index appears in sqlite_stat4 under multiple names */ + continue; + } assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ nIdxCol = pIdx->nKeyCol; @@ -116400,6 +117086,7 @@ static int loadStatTbl( nIdxCol = pIdx->nColumn; } pIdx->nSampleCol = nIdxCol; + pIdx->mxSample = nSample; nByte = sizeof(IndexSample) * nSample; nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ @@ -116439,6 +117126,11 @@ static int loadStatTbl( if( zIndex==0 ) continue; pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); if( pIdx==0 ) continue; + if( pIdx->nSample>=pIdx->mxSample ){ + /* Too many slots used because the same index appears in + ** sqlite_stat4 using multiple names */ + continue; + } /* This next condition is true if data has already been loaded from ** the sqlite_stat4 table. */ nCol = pIdx->nSampleCol; @@ -116482,11 +117174,12 @@ static int loadStat4(sqlite3 *db, const char *zDb){ const Table *pStat4; assert( db->lookaside.bDisable ); - if( (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0 + if( OptimizationEnabled(db, SQLITE_Stat4) + && (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0 && IsOrdinaryTable(pStat4) ){ rc = loadStatTbl(db, - "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx COLLATE nocase", "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb ); @@ -118330,7 +119023,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ if( IsOrdinaryTable(pTable) ){ sqlite3FkDelete(db, pTable); } -#ifndef SQLITE_OMIT_VIRTUAL_TABLE +#ifndef SQLITE_OMIT_VIRTUALTABLE else if( IsVirtual(pTable) ){ sqlite3VtabClear(db, pTable); } @@ -118933,7 +119626,7 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ if( pParse->pNewTrigger ){ sqlite3ErrorMsg(pParse, "cannot use RETURNING in a trigger"); }else{ - assert( pParse->bReturning==0 ); + assert( pParse->bReturning==0 || pParse->ifNotExists ); } pParse->bReturning = 1; pRet = sqlite3DbMallocZero(db, sizeof(*pRet)); @@ -118959,7 +119652,8 @@ SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ pRet->retTStep.pTrig = &pRet->retTrig; pRet->retTStep.pExprList = pList; pHash = &(db->aDb[1].pSchema->trigHash); - assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 || pParse->nErr ); + assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 + || pParse->nErr || pParse->ifNotExists ); if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig) ==&pRet->retTrig ){ sqlite3OomFault(db); @@ -123361,6 +124055,7 @@ SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8 enc){ ** strings is BINARY. */ db->pDfltColl = sqlite3FindCollSeq(db, enc, sqlite3StrBINARY, 0); + sqlite3ExpirePreparedStatements(db, 1); } /* @@ -123832,13 +124527,15 @@ static int tabIsReadOnly(Parse *pParse, Table *pTab){ ** If pTab is writable but other errors have occurred -> return 1. ** If pTab is writable and no prior errors -> return 0; */ -SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, Trigger *pTrigger){ if( tabIsReadOnly(pParse, pTab) ){ sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); return 1; } #ifndef SQLITE_OMIT_VIEW - if( !viewOk && IsView(pTab) ){ + if( IsView(pTab) + && (pTrigger==0 || (pTrigger->bReturning && pTrigger->pNext==0)) + ){ sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); return 1; } @@ -124092,7 +124789,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( goto delete_from_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto delete_from_cleanup; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -124201,7 +124898,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; - if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; + if( sNC.ncFlags & NC_Subquery ) bComplex = 1; wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ @@ -126255,7 +126952,7 @@ static void trimFunc( /* ** The "unknown" function is automatically substituted in place of ** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN -** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used. +** when the SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION compile-time option is used. ** When the "sqlite3" command-line shell is built using this functionality, ** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries ** involving application-defined functions to be examined in a generic @@ -128558,22 +129255,22 @@ static Trigger *fkActionTrigger( if( action==OE_Restrict ){ int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - Token tFrom; - Token tDb; + SrcList *pSrc; Expr *pRaise; - tFrom.z = zFrom; - tFrom.n = nFrom; - tDb.z = db->aDb[iDb].zDbSName; - tDb.n = sqlite3Strlen30(tDb.z); - pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); if( pRaise ){ pRaise->affExpr = OE_Abort; } + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pSrc ){ + assert( pSrc->nSrc==1 ); + pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); + pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), - sqlite3SrcListAppend(pParse, 0, &tDb, &tFrom), + pSrc, pWhere, 0, 0, 0, 0, 0 ); @@ -128789,46 +129486,48 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ +static SQLITE_NOINLINE const char *computeIndexAffStr(sqlite3 *db, Index *pIdx){ + /* The first time a column affinity string for a particular index is + ** required, it is allocated and populated here. It is then stored as + ** a member of the Index structure for subsequent use. + ** + ** The column affinity string will eventually be deleted by + ** sqliteDeleteIndex() when the Index structure itself is cleaned + ** up. + */ + int n; + Table *pTab = pIdx->pTable; + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ - /* The first time a column affinity string for a particular index is - ** required, it is allocated and populated here. It is then stored as - ** a member of the Index structure for subsequent use. - ** - ** The column affinity string will eventually be deleted by - ** sqliteDeleteIndex() when the Index structure itself is cleaned - ** up. - */ - int n; - Table *pTab = pIdx->pTable; - pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); - if( !pIdx->zColAff ){ - sqlite3OomFault(db); - return 0; - } - for(n=0; nnColumn; n++){ - i16 x = pIdx->aiColumn[n]; - char aff; - if( x>=0 ){ - aff = pTab->aCol[x].affinity; - }else if( x==XN_ROWID ){ - aff = SQLITE_AFF_INTEGER; - }else{ - assert( x==XN_EXPR ); - assert( pIdx->bHasExpr ); - assert( pIdx->aColExpr!=0 ); - aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); - } - if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; - pIdx->zColAff[n] = aff; + sqlite3OomFault(db); + return 0; + } + for(n=0; nnColumn; n++){ + i16 x = pIdx->aiColumn[n]; + char aff; + if( x>=0 ){ + aff = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + aff = SQLITE_AFF_INTEGER; + }else{ + assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); } - pIdx->zColAff[n] = 0; + if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; + pIdx->zColAff[n] = aff; } - + pIdx->zColAff[n] = 0; + return pIdx->zColAff; +} +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ + if( !pIdx->zColAff ) return computeIndexAffStr(db, pIdx); return pIdx->zColAff; } + /* ** Compute an affinity string for a table. Space is obtained ** from sqlite3DbMalloc(). The caller is responsible for freeing @@ -129513,7 +130212,7 @@ SQLITE_PRIVATE void sqlite3Insert( /* Cannot insert into a read-only table. */ - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto insert_cleanup; } @@ -129960,7 +130659,7 @@ SQLITE_PRIVATE void sqlite3Insert( } /* Copy the new data already generated. */ - assert( pTab->nNVCol>0 ); + assert( pTab->nNVCol>0 || pParse->nErr>0 ); sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1); #ifndef SQLITE_OMIT_GENERATED_COLUMNS @@ -133323,7 +134022,11 @@ static int sqlite3LoadExtension( /* tag-20210611-1. Some dlopen() implementations will segfault if given ** an oversize filename. Most filesystems have a pathname limit of 4K, ** so limit the extension filename length to about twice that. - ** https://sqlite.org/forum/forumpost/08a0d6d9bf */ + ** https://sqlite.org/forum/forumpost/08a0d6d9bf + ** + ** Later (2023-03-25): Save an extra 6 bytes for the filename suffix. + ** See https://sqlite.org/forum/forumpost/24083b579d. + */ if( nMsg>SQLITE_MAX_PATHLEN ) goto extension_not_found; handle = sqlite3OsDlOpen(pVfs, zFile); @@ -133331,7 +134034,9 @@ static int sqlite3LoadExtension( for(ii=0; iiaDb[iDb].zDbSName; sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3TouchRegister(pParse, pTab->nCol+regRow); sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); sqlite3VdbeLoadString(v, regResult, pTab->zName); assert( IsOrdinaryTable(pTab) ); @@ -135867,7 +136572,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** regRow..regRow+n. If any of the child key values are NULL, this ** row cannot cause an FK violation. Jump directly to addrOk in ** this case. */ - if( regRow+pFK->nCol>pParse->nMem ) pParse->nMem = regRow+pFK->nCol; + sqlite3TouchRegister(pParse, regRow + pFK->nCol); for(j=0; jnCol; j++){ int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom; sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j); @@ -135996,6 +136701,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iDb>=0 && i!=iDb ) continue; sqlite3CodeVerifySchema(pParse, i); + pParse->okConstFactor = 0; /* tag-20230327-1 */ /* Do an integrity check of the B-Tree ** @@ -136031,7 +136737,7 @@ SQLITE_PRIVATE void sqlite3Pragma( aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); + sqlite3TouchRegister(pParse, 8+mxIdx); sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ @@ -136181,15 +136887,29 @@ SQLITE_PRIVATE void sqlite3Pragma( labelOk = sqlite3VdbeMakeLabel(pParse); if( pCol->notNull ){ /* (1) NOT NULL columns may not contain a NULL */ + int jmp3; int jmp2 = sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); - sqlite3VdbeChangeP5(v, 0x0f); VdbeCoverage(v); + if( p1<0 ){ + sqlite3VdbeChangeP5(v, 0x0f); /* INT, REAL, TEXT, or BLOB */ + jmp3 = jmp2; + }else{ + sqlite3VdbeChangeP5(v, 0x0d); /* INT, TEXT, or BLOB */ + /* OP_IsType does not detect NaN values in the database file + ** which should be treated as a NULL. So if the header type + ** is REAL, we have to load the actual data using OP_Column + ** to reliably determine if the value is a NULL. */ + sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); + VdbeCoverage(v); + } zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pCol->zCnName); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); if( doTypeCheck ){ sqlite3VdbeGoto(v, labelError); sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeJumpHere(v, jmp3); }else{ /* VDBE byte code will fall thru */ } @@ -136297,7 +137017,7 @@ SQLITE_PRIVATE void sqlite3Pragma( int jmp7; sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur+j, 3); jmp7 = sqlite3VdbeAddOp3(v, OP_Eq, 3, 0, r1+pIdx->nColumn-1); - VdbeCoverage(v); + VdbeCoverageNeverNull(v); sqlite3VdbeLoadString(v, 3, "rowid not at end-of-record for row "); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); @@ -137503,7 +138223,9 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl #else encoding = SQLITE_UTF8; #endif - if( db->nVdbeActive>0 && encoding!=ENC(db) ){ + if( db->nVdbeActive>0 && encoding!=ENC(db) + && (db->mDbFlags & DBFLAG_Vacuum)==0 + ){ rc = SQLITE_LOCKED; goto initone_error_out; }else{ @@ -137897,7 +138619,11 @@ static int sqlite3Prepare( sParse.db = db; sParse.pReprepare = pReprepare; assert( ppStmt && *ppStmt==0 ); - if( db->mallocFailed ) sqlite3ErrorMsg(&sParse, "out of memory"); + if( db->mallocFailed ){ + sqlite3ErrorMsg(&sParse, "out of memory"); + db->errCode = rc = SQLITE_NOMEM; + goto end_prepare; + } assert( sqlite3_mutex_held(db->mutex) ); /* For a long-term use prepared statement avoid the use of @@ -138986,7 +139712,7 @@ static void pushOntoSorter( ** (2) All output columns are included in the sort record. In that ** case regData==regOrigData. ** (3) Some output columns are omitted from the sort record due to - ** the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the + ** the SQLITE_ENABLE_SORTER_REFERENCES optimization, or due to the ** SQLITE_ECEL_OMITREF optimization, or due to the ** SortCtx.pDeferredRowLoad optimiation. In any of these cases ** regOrigData is 0 to prevent this routine from trying to copy @@ -140587,7 +141313,7 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( assert( (pSelect->selFlags & SF_Resolved)!=0 ); assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); - if( db->mallocFailed ) return; + if( db->mallocFailed || IN_RENAME_OBJECT ) return; while( pSelect->pPrior ) pSelect = pSelect->pPrior; a = pSelect->pEList->a; memset(&sNC, 0, sizeof(sNC)); @@ -140632,18 +141358,16 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( break; } } - } - } - if( zType ){ - i64 m = sqlite3Strlen30(zType); - n = sqlite3Strlen30(pCol->zCnName); - pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); - if( pCol->zCnName ){ - memcpy(&pCol->zCnName[n+1], zType, m+1); - pCol->colFlags |= COLFLAG_HASTYPE; - }else{ - testcase( pCol->colFlags & COLFLAG_HASTYPE ); - pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); + } + } + if( zType ){ + i64 m = sqlite3Strlen30(zType); + n = sqlite3Strlen30(pCol->zCnName); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); + if( pCol->zCnName ){ + memcpy(&pCol->zCnName[n+1], zType, m+1); + pCol->colFlags |= COLFLAG_HASTYPE; } } pColl = sqlite3ExprCollSeq(pParse, p); @@ -142131,7 +142855,9 @@ static Expr *substExpr( sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ sqlite3 *db = pSubst->pParse->db; - if( pSubst->isOuterJoin ){ + if( pSubst->isOuterJoin + && (pCopy->op!=TK_COLUMN || pCopy->iTable!=pSubst->iNewTable) + ){ memset(&ifNullRow, 0, sizeof(ifNullRow)); ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; @@ -142508,8 +143234,7 @@ static int compoundHasDifferentAffinities(Select *p){ ** query or there are no RIGHT or FULL JOINs in any arm ** of the subquery. (This is a duplicate of condition (27b).) ** (17h) The corresponding result set expressions in all arms of the -** compound must have the same affinity. (See restriction (9) -** on the push-down optimization.) +** compound must have the same affinity. ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -143377,10 +144102,24 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ ** or EXCEPT, then all of the result set columns for all arms of ** the compound must use the BINARY collating sequence. ** -** (9) If the subquery is a compound, then all arms of the compound must -** have the same affinity. (This is the same as restriction (17h) -** for query flattening.) +** (9) All three of the following are true: +** +** (9a) The WHERE clause expression originates in the ON or USING clause +** of a join (either an INNER or an OUTER join), and +** +** (9b) The subquery is to the right of the ON/USING clause ** +** (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING +** clause and the subquery. +** +** Without this restriction, the push-down optimization might move +** the ON/USING filter expression from the left side of a RIGHT JOIN +** over to the right side, which leads to incorrect answers. See +** also restriction (6) in sqlite3ExprIsSingleTableConstraint(). +** +** (10) The inner query is not the right-hand table of a RIGHT JOIN. +** +** (11) The subquery is not a VALUES clause ** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. @@ -143389,13 +144128,20 @@ static int pushDownWhereTerms( Parse *pParse, /* Parse context (for malloc() and error reporting) */ Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ Expr *pWhere, /* The WHERE clause of the outer query */ - SrcItem *pSrc /* The subquery term of the outer FROM clause */ + SrcList *pSrcList, /* The complete from clause of the outer query */ + int iSrc /* Which FROM clause term to try to push into */ ){ Expr *pNew; + SrcItem *pSrc; /* The subquery FROM term into which WHERE is pushed */ int nChng = 0; + pSrc = &pSrcList->a[iSrc]; if( pWhere==0 ) return 0; - if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0; - if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0; + if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ){ + return 0; /* restrictions (2) and (11) */ + } + if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ){ + return 0; /* restrictions (10) */ + } if( pSubq->pPrior ){ Select *pSel; @@ -143411,9 +144157,6 @@ static int pushDownWhereTerms( if( pSel->pWin ) return 0; /* restriction (6b) */ #endif } - if( compoundHasDifferentAffinities(pSubq) ){ - return 0; /* restriction (9) */ - } if( notUnionAll ){ /* If any of the compound arms are connected using UNION, INTERSECT, ** or EXCEPT, then we must ensure that none of the columns use a @@ -143453,11 +144196,28 @@ static int pushDownWhereTerms( return 0; /* restriction (3) */ } while( pWhere->op==TK_AND ){ - nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrc); + nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrcList, iSrc); pWhere = pWhere->pLeft; } -#if 0 /* Legacy code. Checks now done by sqlite3ExprIsTableConstraint() */ +#if 0 /* These checks now done by sqlite3ExprIsSingleTableConstraint() */ + if( ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) /* (9a) */ + && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0 /* Fast pre-test of (9c) */ + ){ + int jj; + for(jj=0; jjw.iJoin==pSrcList->a[jj].iCursor ){ + /* If we reach this point, both (9a) and (9b) are satisfied. + ** The following loop checks (9c): + */ + for(jj++; jja[jj].fg.jointype & JT_RIGHT)!=0 ){ + return 0; /* restriction (9) */ + } + } + } + } + } if( isLeftJoin && (ExprHasProperty(pWhere,EP_OuterON)==0 || pWhere->w.iJoin!=iCursor) @@ -143471,7 +144231,7 @@ static int pushDownWhereTerms( } #endif - if( sqlite3ExprIsTableConstraint(pWhere, pSrc) ){ + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){ nChng++; pSubq->selFlags |= SF_PushDown; while( pSubq ){ @@ -143505,6 +144265,78 @@ static int pushDownWhereTerms( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** Check to see if a subquery contains result-set columns that are +** never used. If it does, change the value of those result-set columns +** to NULL so that they do not cause unnecessary work to compute. +** +** Return the number of column that were changed to NULL. +*/ +static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ + int nCol; + Select *pSub; /* The subquery to be simplified */ + Select *pX; /* For looping over compound elements of pSub */ + Table *pTab; /* The table that describes the subquery */ + int j; /* Column number */ + int nChng = 0; /* Number of columns converted to NULL */ + Bitmask colUsed; /* Columns that may not be NULLed out */ + + assert( pItem!=0 ); + if( pItem->fg.isCorrelated || pItem->fg.isCte ){ + return 0; + } + assert( pItem->pTab!=0 ); + pTab = pItem->pTab; + assert( pItem->pSelect!=0 ); + pSub = pItem->pSelect; + assert( pSub->pEList->nExpr==pTab->nCol ); + if( (pSub->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ + testcase( pSub->selFlags & SF_Distinct ); + testcase( pSub->selFlags & SF_Aggregate ); + return 0; + } + for(pX=pSub; pX; pX=pX->pPrior){ + if( pX->pPrior && pX->op!=TK_ALL ){ + /* This optimization does not work for compound subqueries that + ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ + return 0; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pX->pWin ){ + /* This optimization does not work for subqueries that use window + ** functions. */ + return 0; + } +#endif + } + colUsed = pItem->colUsed; + if( pSub->pOrderBy ){ + ExprList *pList = pSub->pOrderBy; + for(j=0; jnExpr; j++){ + u16 iCol = pList->a[j].u.x.iOrderByCol; + if( iCol>0 ){ + iCol--; + colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + } + } + nCol = pTab->nCol; + for(j=0; jpPrior) { + Expr *pY = pX->pEList->a[j].pExpr; + if( pY->op==TK_NULL ) continue; + pY->op = TK_NULL; + ExprClearProperty(pY, EP_Skip|EP_Unlikely); + pX->selFlags |= SF_PushDown; + nChng++; + } + } + return nChng; +} + + /* ** The pFunc is the only aggregate function in the query. Check to see ** if the query is a candidate for the min/max optimization. @@ -144651,12 +145483,13 @@ static void optimizeAggregateUseOfIndexedExpr( assert( pSelect->pGroupBy!=0 ); pAggInfo->nColumn = pAggInfo->nAccumulator; if( ALWAYS(pAggInfo->nSortingColumn>0) ){ - if( pAggInfo->nColumn==0 ){ - pAggInfo->nSortingColumn = pSelect->pGroupBy->nExpr; - }else{ - pAggInfo->nSortingColumn = - pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1; + int mx = pSelect->pGroupBy->nExpr - 1; + int j, k; + for(j=0; jnColumn; j++){ + k = pAggInfo->aCol[j].iSorterColumn; + if( k>mx ) mx = k; } + pAggInfo->nSortingColumn = mx+1; } analyzeAggFuncArgs(pAggInfo, pNC); #if TREETRACE_ENABLED @@ -144690,11 +145523,13 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; pAggInfo = pExpr->pAggInfo; - assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + if( NEVER(pExpr->iAgg>=pAggInfo->nColumn) ) return WRC_Continue; + assert( pExpr->iAgg>=0 ); pCol = &pAggInfo->aCol[pExpr->iAgg]; pExpr->op = TK_AGG_COLUMN; pExpr->iTable = pCol->iTable; pExpr->iColumn = pCol->iColumn; + ExprClearProperty(pExpr, EP_Skip|EP_Collate); return WRC_Prune; } @@ -145048,7 +145883,6 @@ static void agginfoFree(sqlite3 *db, AggInfo *p){ sqlite3DbFreeNN(db, p); } -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION /* ** Attempt to transform a query of the form ** @@ -145076,6 +145910,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ if( p->pWhere ) return 0; + if( p->pHaving ) return 0; if( p->pGroupBy ) return 0; if( p->pOrderBy ) return 0; pExpr = p->pEList->a[0].pExpr; @@ -145095,7 +145930,8 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ if( pSub->pWhere ) return 0; /* No WHERE clause */ if( pSub->pLimit ) return 0; /* No LIMIT clause */ if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ - pSub = pSub->pPrior; /* Repeat over compound */ + assert( pSub->pHaving==0 ); /* Due to the previous */ + pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); /* If we reach this point then it is OK to perform the transformation */ @@ -145138,7 +145974,6 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ #endif return 1; } -#endif /* SQLITE_COUNTOFVIEW_OPTIMIZATION */ /* ** If any term of pSrc, or any SF_NestedFrom sub-query, is not the same @@ -145394,7 +146229,7 @@ SQLITE_PRIVATE int sqlite3Select( pTabList->a[0].fg.jointype & JT_LTORJ); } - /* No futher action if this term of the FROM clause is no a subquery */ + /* No futher action if this term of the FROM clause is not a subquery */ if( pSub==0 ) continue; /* Catch mismatch in the declared columns of a view and the number of @@ -145527,14 +146362,12 @@ SQLITE_PRIVATE int sqlite3Select( TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); } -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) && countOfViewOptimization(pParse, p) ){ if( db->mallocFailed ) goto select_end; pTabList = p->pSrc; } -#endif /* For each term in the FROM clause, do two things: ** (1) Authorized unreferenced tables @@ -145593,7 +146426,7 @@ SQLITE_PRIVATE int sqlite3Select( if( OptimizationEnabled(db, SQLITE_PushDown) && (pItem->fg.isCte==0 || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2)) - && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem) + && pushDownWhereTerms(pParse, pSub, p->pWhere, pTabList, i) ){ #if TREETRACE_ENABLED if( sqlite3TreeTrace & 0x4000 ){ @@ -145607,6 +146440,22 @@ SQLITE_PRIVATE int sqlite3Select( TREETRACE(0x4000,pParse,p,("Push-down not possible\n")); } + /* Convert unused result columns of the subquery into simple NULL + ** expressions, to avoid unneeded searching and computation. + */ + if( OptimizationEnabled(db, SQLITE_NullUnusedCols) + && disableUnusedSubqueryResultColumns(pItem) + ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, + ("Change unused result columns to NULL for subquery %d:\n", + pSub->selId)); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + } + zSavedAuthContext = pParse->zAuthContext; pParse->zAuthContext = pItem->zName; @@ -146893,6 +147742,7 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( }else{ assert( !db->init.busy ); sqlite3CodeVerifySchema(pParse, iDb); + VVA_ONLY( pParse->ifNotExists = 1; ) } goto trigger_cleanup; } @@ -148143,6 +148993,9 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask( Trigger *p; assert( isNew==1 || isNew==0 ); + if( IsView(pTab) ){ + return 0xffffffff; + } for(p=pTrigger; p; p=p->pNext){ if( p->op==op && (tr_tm&p->tr_tm) @@ -148577,7 +149430,7 @@ SQLITE_PRIVATE void sqlite3Update( if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto update_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto update_cleanup; } @@ -148896,12 +149749,22 @@ SQLITE_PRIVATE void sqlite3Update( /* Begin the database scan. ** ** Do not consider a single-pass strategy for a multi-row update if - ** there are any triggers or foreign keys to process, or rows may - ** be deleted as a result of REPLACE conflict handling. Any of these - ** things might disturb a cursor being used to scan through the table - ** or index, causing a single-pass approach to malfunction. */ + ** there is anything that might disrupt the cursor being used to do + ** the UPDATE: + ** (1) This is a nested UPDATE + ** (2) There are triggers + ** (3) There are FOREIGN KEY constraints + ** (4) There are REPLACE conflict handlers + ** (5) There are subqueries in the WHERE clause + */ flags = WHERE_ONEPASS_DESIRED; - if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){ + if( !pParse->nested + && !pTrigger + && !hasFK + && !chngKey + && !bReplace + && (sNC.ncFlags & NC_Subquery)==0 + ){ flags |= WHERE_ONEPASS_MULTIROW; } pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,0,0,flags,iIdxCur); @@ -150866,7 +151729,9 @@ static int vtabCallConstructor( sCtx.pPrior = db->pVtabCtx; sCtx.bDeclared = 0; db->pVtabCtx = &sCtx; + pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); assert( sCtx.pTab==pTab ); @@ -151356,7 +152221,10 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ break; } if( xMethod && pVTab->iSavepoint>iSavepoint ){ + u64 savedFlags = (db->flags & SQLITE_Defensive); + db->flags &= ~(u64)SQLITE_Defensive; rc = xMethod(pVTab->pVtab, iSavepoint); + db->flags |= savedFlags; } sqlite3VtabUnlock(pVTab); } @@ -151585,6 +152453,10 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ p->pVTable->eVtabRisk = SQLITE_VTABRISK_High; break; } + case SQLITE_VTAB_USES_ALL_SCHEMAS: { + p->pVTable->bAllSchemas = 1; + break; + } default: { rc = SQLITE_MISUSE_BKPT; break; @@ -152358,9 +153230,9 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ /* ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was -** defined at compile-time. If it is not a no-op, a single OP_Explain opcode -** is added to the output to describe the table scan strategy in pLevel. +** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG +** was defined at compile-time. If it is not a no-op, a single OP_Explain +** opcode is added to the output to describe the table scan strategy in pLevel. ** ** If an OP_Explain opcode is added to the VM, its address is returned. ** Otherwise, if no OP_Explain is coded, zero is returned. @@ -152372,8 +153244,8 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ int ret = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( sqlite3ParseToplevel(pParse)->explain==2 ) +#if !defined(SQLITE_DEBUG) + if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { SrcItem *pItem = &pTabList->a[pLevel->iFrom]; @@ -152539,27 +153411,29 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( WhereLevel *pLvl, /* Level to add scanstatus() entry for */ int addrExplain /* Address of OP_Explain (or 0) */ ){ - const char *zObj = 0; - WhereLoop *pLoop = pLvl->pWLoop; - int wsFlags = pLoop->wsFlags; - int viaCoroutine = 0; - - if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ - zObj = pLoop->u.btree.pIndex->zName; - }else{ - zObj = pSrclist->a[pLvl->iFrom].zName; - viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; - } - sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj - ); + if( IS_STMT_SCANSTATUS( sqlite3VdbeDb(v) ) ){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + int wsFlags = pLoop->wsFlags; + int viaCoroutine = 0; - if( viaCoroutine==0 ){ - if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ - sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; } - if( wsFlags & WHERE_INDEXED ){ - sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); + + if( viaCoroutine==0 ){ + if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } + if( wsFlags & WHERE_INDEXED ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + } } } } @@ -153256,11 +154130,12 @@ static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ */ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ int rc = WRC_Continue; + int reg; struct CCurHint *pHint = pWalker->u.pCCurHint; if( pExpr->op==TK_COLUMN ){ if( pExpr->iTable!=pHint->iTabCur ){ - int reg = ++pWalker->pParse->nMem; /* Register for column value */ - sqlite3ExprCode(pWalker->pParse, pExpr, reg); + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); pExpr->op = TK_REGISTER; pExpr->iTable = reg; }else if( pHint->pIdx!=0 ){ @@ -153268,15 +154143,15 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn); assert( pExpr->iColumn>=0 ); } - }else if( pExpr->op==TK_AGG_FUNCTION ){ - /* An aggregate function in the WHERE clause of a query means this must - ** be a correlated sub-query, and expression pExpr is an aggregate from - ** the parent context. Do not walk the function arguments in this case. - ** - ** todo: It should be possible to replace this node with a TK_REGISTER - ** expression, as the result of the expression must be stored in a - ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + }else if( pExpr->pAggInfo ){ rc = WRC_Prune; + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; + }else if( pExpr->op==TK_TRUEFALSE ){ + /* Do not walk disabled expressions. tag-20230504-1 */ + return WRC_Prune; } return rc; } @@ -153378,7 +154253,7 @@ static void codeCursorHint( } if( pExpr!=0 ){ sWalker.xExprCallback = codeCursorHintFixExpr; - sqlite3WalkExpr(&sWalker, pExpr); + if( pParse->nErr==0 ) sqlite3WalkExpr(&sWalker, pExpr); sqlite3VdbeAddOp4(v, OP_CursorHint, (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, (const char*)pExpr, P4_EXPR); @@ -154172,7 +155047,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** guess. */ addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, (pIdx->aiRowLogEst[0]+9)/10); - if( pRangeStart ){ + if( pRangeStart || pRangeEnd ){ sqlite3VdbeChangeP5(v, 1); sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1); addrSeekScan = 0; @@ -154213,16 +155088,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pLevel->p2==0 ); if( pRangeEnd ){ Expr *pRight = pRangeEnd->pExpr->pRight; - if( addrSeekScan ){ - /* For a seek-scan that has a range on the lowest term of the index, - ** we have to make the top of the loop be code that sets the end - ** condition of the range. Otherwise, the OP_SeekScan might jump - ** over that initialization, leaving the range-end value set to the - ** range-start value, resulting in a wrong answer. - ** See ticket 5981a8c041a3c2f3 (2021-11-02). - */ - pLevel->p2 = sqlite3VdbeCurrentAddr(v); - } + assert( addrSeekScan==0 ); codeExprOrVector(pParse, pRight, regBase+nEq, nTop); whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); if( (pRangeEnd->wtFlags & TERM_VNULL)==0 @@ -154256,7 +155122,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff); /* Top of the loop body */ - if( pLevel->p2==0 ) pLevel->p2 = sqlite3VdbeCurrentAddr(v); + pLevel->p2 = sqlite3VdbeCurrentAddr(v); /* Check if the index cursor is past the end of the range. */ if( nConstraint ){ @@ -156253,7 +157119,7 @@ static void exprAnalyze( && 0==sqlite3ExprCanBeNull(pLeft) ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); - pExpr->op = TK_TRUEFALSE; + pExpr->op = TK_TRUEFALSE; /* See tag-20230504-1 */ pExpr->u.zToken = "false"; ExprSetProperty(pExpr, EP_IsFalse); pTerm->prereqAll = 0; @@ -156898,9 +157764,12 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( pRhs = sqlite3PExpr(pParse, TK_UPLUS, sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); - if( pItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ){ + if( pItem->fg.jointype & (JT_LEFT|JT_RIGHT) ){ + testcase( pItem->fg.jointype & JT_LEFT ); /* testtag-20230227a */ + testcase( pItem->fg.jointype & JT_RIGHT ); /* testtag-20230227b */ joinType = EP_OuterON; }else{ + testcase( pItem->fg.jointype & JT_LTORJ ); /* testtag-20230227c */ joinType = EP_InnerON; } sqlite3SetJoinExpr(pTerm, pItem->iCursor, joinType); @@ -157743,7 +158612,7 @@ static void explainAutomaticIndex( int bPartial, /* True if pIdx is a partial index */ int *pAddrExplain /* OUT: Address of OP_Explain */ ){ - if( pParse->explain!=2 ){ + if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){ Table *pTab = pIdx->pTable; const char *zSep = ""; char *zText = 0; @@ -157782,8 +158651,7 @@ static void explainAutomaticIndex( */ static SQLITE_NOINLINE void constructAutomaticIndex( Parse *pParse, /* The parsing context */ - const WhereClause *pWC, /* The WHERE clause */ - const SrcItem *pSrc, /* The FROM clause term to get the next index */ + WhereClause *pWC, /* The WHERE clause */ const Bitmask notReady, /* Mask of cursors that are not available */ WhereLevel *pLevel /* Write new index here */ ){ @@ -157804,10 +158672,12 @@ static SQLITE_NOINLINE void constructAutomaticIndex( char *zNotUsed; /* Extra space on the end of pIdx */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ - u8 sentWarning = 0; /* True if a warnning has been issued */ + u8 sentWarning = 0; /* True if a warning has been issued */ + u8 useBloomFilter = 0; /* True to also add a Bloom filter */ Expr *pPartial = 0; /* Partial Index Expression */ int iContinue = 0; /* Jump here to skip excluded rows */ - SrcItem *pTabItem; /* FROM clause term being indexed */ + SrcList *pTabList; /* The complete FROM clause */ + SrcItem *pSrc; /* The FROM clause term to get the next index */ int addrCounter = 0; /* Address where integer counter is initialized */ int regBase; /* Array of registers where record is assembled */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS @@ -157823,6 +158693,8 @@ static SQLITE_NOINLINE void constructAutomaticIndex( /* Count the number of columns that will be added to the index ** and used to match WHERE clause constraints */ nKeyCol = 0; + pTabList = pWC->pWInfo->pTabList; + pSrc = &pTabList->a[pLevel->iFrom]; pTable = pSrc->pTab; pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; @@ -157833,7 +158705,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** WHERE clause (or the ON clause of a LEFT join) that constrain which ** rows of the target table (pSrc) that can be used. */ if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsTableConstraint(pExpr, pSrc) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom) ){ pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); @@ -157874,7 +158746,11 @@ static SQLITE_NOINLINE void constructAutomaticIndex( ** original table changes and the index and table cannot both be used ** if they go out of sync. */ - extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + if( IsView(pTable) ){ + extraCols = ALLBITS; + }else{ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + } mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); @@ -157910,6 +158786,16 @@ static SQLITE_NOINLINE void constructAutomaticIndex( assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; n++; + if( ALWAYS(pX->pLeft!=0) + && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT + ){ + /* TUNING: only use a Bloom filter on an automatic index + ** if one or more key columns has the ability to hold numeric + ** values, since strings all have the same hash in the Bloom + ** filter implementation and hence a Bloom filter on a text column + ** is not usually helpful. */ + useBloomFilter = 1; + } } } } @@ -157942,20 +158828,21 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "for %s", pTable->zName)); - if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){ + sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel); pLevel->regFilter = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); } /* Fill the automatic index with content */ - pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; - if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; + assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] ); + if( pSrc->fg.viaCoroutine ){ + int regYield = pSrc->regReturn; addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); - VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pSrc->pTab->zName)); }else{ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); } @@ -157976,14 +158863,14 @@ static SQLITE_NOINLINE void constructAutomaticIndex( sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); - if( pTabItem->fg.viaCoroutine ){ + if( pSrc->fg.viaCoroutine ){ sqlite3VdbeChangeP2(v, addrCounter, regBase+n); testcase( pParse->db->mallocFailed ); assert( pLevel->iIdxCur>0 ); translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, - pTabItem->regResult, pLevel->iIdxCur); + pSrc->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); - pTabItem->fg.viaCoroutine = 0; + pSrc->fg.viaCoroutine = 0; }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); @@ -158035,6 +158922,10 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( Vdbe *v = pParse->pVdbe; /* VDBE under construction */ WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */ int iCur; /* Cursor for table getting the filter */ + IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */ + + saved_pIdxEpr = pParse->pIdxEpr; + pParse->pIdxEpr = 0; assert( pLoop!=0 ); assert( v!=0 ); @@ -158042,9 +158933,11 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); do{ + const SrcList *pTabList; const SrcItem *pItem; const Table *pTab; u64 sz; + int iSrc; sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel); addrCont = sqlite3VdbeMakeLabel(pParse); iCur = pLevel->iTabCur; @@ -158058,7 +158951,9 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( ** testing complicated. By basing the blob size on the value in the ** sqlite_stat1 table, testing is much easier. */ - pItem = &pWInfo->pTabList->a[pLevel->iFrom]; + pTabList = pWInfo->pTabList; + iSrc = pLevel->iFrom; + pItem = &pTabList->a[iSrc]; assert( pItem!=0 ); pTab = pItem->pTab; assert( pTab!=0 ); @@ -158075,7 +158970,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( for(pTerm=pWInfo->sWC.a; pTermpExpr; if( (pTerm->wtFlags & TERM_VIRTUAL)==0 - && sqlite3ExprIsTableConstraint(pExpr, pItem) + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc) ){ sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); } @@ -158091,9 +158986,8 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( int r1 = sqlite3GetTempRange(pParse, n); int jj; for(jj=0; jjaiColumn[jj]; assert( pIdx->pTable==pItem->pTab ); - sqlite3ExprCodeGetColumnOfTable(v, pIdx->pTable, iCur, iCol,r1+jj); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); } sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); sqlite3ReleaseTempRange(pParse, r1, n); @@ -158124,6 +159018,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( } }while( iLevel < pWInfo->nLevel ); sqlite3VdbeJumpHere(v, addrOnce); + pParse->pIdxEpr = saved_pIdxEpr; } @@ -158379,6 +159274,9 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); } } + if( pTab->u.vtab.p->bAllSchemas ){ + sqlite3VtabUsesAllSchemas(pParse); + } sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; return rc; @@ -158423,6 +159321,7 @@ static int whereKeyStats( assert( pIdx->nSample>0 ); assert( pRec->nField>0 ); + /* Do a binary search to find the first sample greater than or equal ** to pRec. If pRec contains a single field, the set of samples to search ** is simply the aSample[] array. If the samples in aSample[] contain more @@ -158467,7 +159366,12 @@ static int whereKeyStats( ** it is extended to two fields. The duplicates that this creates do not ** cause any problems. */ - nField = MIN(pRec->nField, pIdx->nSample); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nField = pIdx->nKeyCol; + }else{ + nField = pIdx->nColumn; + } + nField = MIN(pRec->nField, nField); iCol = 0; iSample = pIdx->nSample * nField; do{ @@ -158903,7 +159807,7 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); assert( pLower || pUpper ); #endif - assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); @@ -161004,8 +161908,6 @@ SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ return pHidden->eDistinct; } -#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ - && !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** Cause the prepared statement that is associated with a call to ** xBestIndex to potentially use all schemas. If the statement being @@ -161015,9 +161917,7 @@ SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ ** ** This is used by the (built-in) sqlite_dbpage virtual table. */ -SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(sqlite3_index_info *pIdxInfo){ - HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; - Parse *pParse = pHidden->pParse; +SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse *pParse){ int nDb = pParse->db->nDb; int i; for(i=0; iisOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } + if( pWInfo->pSelect->pOrderBy + && pWInfo->nOBSat > pWInfo->pSelect->pOrderBy->nExpr ){ + pWInfo->nOBSat = pWInfo->pSelect->pOrderBy->nExpr; + } }else{ pWInfo->revMask = pFrom->revLoop; if( pWInfo->nOBSat<=0 ){ @@ -162406,6 +163309,13 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ ** at most a single row. ** 4) The table must not be referenced by any part of the query apart ** from its own USING or ON clause. +** 5) The table must not have an inner-join ON or USING clause if there is +** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause +** might move from the right side to the left side of the RIGHT JOIN. +** Note: Due to (2), this condition can only arise if the table is +** the right-most table of a subquery that was flattened into the +** main query and that subquery was the right-hand operand of an +** inner join that held an ON or USING clause. ** ** For example, given: ** @@ -162431,6 +163341,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( ){ int i; Bitmask tabUsed; + int hasRightJoin; /* Preconditions checked by the caller */ assert( pWInfo->nLevel>=2 ); @@ -162445,6 +163356,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( if( pWInfo->pOrderBy ){ tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy); } + hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0; for(i=pWInfo->nLevel-1; i>=1; i--){ WhereTerm *pTerm, *pEnd; SrcItem *pItem; @@ -162467,6 +163379,12 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( break; } } + if( hasRightJoin + && ExprHasProperty(pTerm->pExpr, EP_InnerON) + && pTerm->pExpr->w.iJoin==pItem->iCursor + ){ + break; /* restriction (5) */ + } } if( pTerm drop loop %c not used\n", pLoop->cId)); @@ -162866,22 +163784,45 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } if( pParse->nErr ) goto whereBeginError; - /* Special case: WHERE terms that do not refer to any tables in the join - ** (constant expressions). Evaluate each such term, and jump over all the - ** generated code if the result is not true. + /* The False-WHERE-Term-Bypass optimization: ** - ** Do not do this if the expression contains non-deterministic functions - ** that are not within a sub-select. This is not strictly required, but - ** preserves SQLite's legacy behaviour in the following two cases: + ** If there are WHERE terms that are false, then no rows will be output, + ** so skip over all of the code generated here. ** - ** FROM ... WHERE random()>0; -- eval random() once per row - ** FROM ... WHERE (SELECT random())>0; -- eval random() once overall + ** Conditions: + ** + ** (1) The WHERE term must not refer to any tables in the join. + ** (2) The term must not come from an ON clause on the + ** right-hand side of a LEFT or FULL JOIN. + ** (3) The term must not come from an ON clause, or there must be + ** no RIGHT or FULL OUTER joins in pTabList. + ** (4) If the expression contains non-deterministic functions + ** that are not within a sub-select. This is not required + ** for correctness but rather to preserves SQLite's legacy + ** behaviour in the following two cases: + ** + ** WHERE random()>0; -- eval random() once per row + ** WHERE (SELECT random())>0; -- eval random() just once overall + ** + ** Note that the Where term need not be a constant in order for this + ** optimization to apply, though it does need to be constant relative to + ** the current subquery (condition 1). The term might include variables + ** from outer queries so that the value of the term changes from one + ** invocation of the current subquery to the next. */ for(ii=0; iinBase; ii++){ - WhereTerm *pT = &sWLB.pWC->a[ii]; + WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */ + Expr *pX; /* The expression of pT */ if( pT->wtFlags & TERM_VIRTUAL ) continue; - if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){ - sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL); + pX = pT->pExpr; + assert( pX!=0 ); + assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) ); + if( pT->prereqAll==0 /* Conditions (1) and (2) */ + && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */ + && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */ + && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 ) + ){ + sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL); pT->wtFlags |= TERM_CODED; } } @@ -163124,7 +164065,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( assert( n<=pTab->nCol ); } #ifdef SQLITE_ENABLE_CURSOR_HINTS - if( pLoop->u.btree.pIndex!=0 ){ + if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); }else #endif @@ -163261,11 +164202,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3VdbeJumpHere(v, iOnce); } } + assert( pTabList == pWInfo->pTabList ); if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){ if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){ #ifndef SQLITE_OMIT_AUTOMATIC_INDEX - constructAutomaticIndex(pParse, &pWInfo->sWC, - &pTabList->a[pLevel->iFrom], notReady, pLevel); + constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel); #endif }else{ sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady); @@ -163582,7 +164523,8 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ k = pLevel->addrBody + 1; #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeAddopTrace ){ - printf("TRANSLATE opcodes in range %d..%d\n", k, last-1); + printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n", + pLevel->iTabCur, pLevel->iIdxCur, k, last-1); } /* Proof that the "+1" on the k value above is safe */ pOp = sqlite3VdbeGetOp(v, k - 1); @@ -164457,6 +165399,7 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ } /* no break */ deliberate_fall_through + case TK_IF_NULL_ROW: case TK_AGG_FUNCTION: case TK_COLUMN: { int iCol = -1; @@ -167285,18 +168228,18 @@ typedef union { #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 576 -#define YYNRULE 405 -#define YYNRULE_WITH_ACTION 342 +#define YYNSTATE 575 +#define YYNRULE 403 +#define YYNRULE_WITH_ACTION 340 #define YYNTOKEN 185 -#define YY_MAX_SHIFT 575 -#define YY_MIN_SHIFTREDUCE 835 -#define YY_MAX_SHIFTREDUCE 1239 -#define YY_ERROR_ACTION 1240 -#define YY_ACCEPT_ACTION 1241 -#define YY_NO_ACTION 1242 -#define YY_MIN_REDUCE 1243 -#define YY_MAX_REDUCE 1647 +#define YY_MAX_SHIFT 574 +#define YY_MIN_SHIFTREDUCE 833 +#define YY_MAX_SHIFTREDUCE 1235 +#define YY_ERROR_ACTION 1236 +#define YY_ACCEPT_ACTION 1237 +#define YY_NO_ACTION 1238 +#define YY_MIN_REDUCE 1239 +#define YY_MAX_REDUCE 1641 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -167363,218 +168306,218 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2098) +#define YY_ACTTAB_COUNT (2096) static const YYACTIONTYPE yy_action[] = { /* 0 */ 568, 208, 568, 118, 115, 229, 568, 118, 115, 229, - /* 10 */ 568, 1314, 377, 1293, 408, 562, 562, 562, 568, 409, - /* 20 */ 378, 1314, 1276, 41, 41, 41, 41, 208, 1526, 71, - /* 30 */ 71, 971, 419, 41, 41, 491, 303, 279, 303, 972, - /* 40 */ 397, 71, 71, 125, 126, 80, 1217, 1217, 1050, 1053, - /* 50 */ 1040, 1040, 123, 123, 124, 124, 124, 124, 476, 409, - /* 60 */ 1241, 1, 1, 575, 2, 1245, 550, 118, 115, 229, - /* 70 */ 317, 480, 146, 480, 524, 118, 115, 229, 529, 1327, - /* 80 */ 417, 523, 142, 125, 126, 80, 1217, 1217, 1050, 1053, - /* 90 */ 1040, 1040, 123, 123, 124, 124, 124, 124, 118, 115, + /* 10 */ 568, 1310, 377, 1289, 408, 562, 562, 562, 568, 409, + /* 20 */ 378, 1310, 1272, 41, 41, 41, 41, 208, 1520, 71, + /* 30 */ 71, 969, 419, 41, 41, 491, 303, 279, 303, 970, + /* 40 */ 397, 71, 71, 125, 126, 80, 1212, 1212, 1047, 1050, + /* 50 */ 1037, 1037, 123, 123, 124, 124, 124, 124, 476, 409, + /* 60 */ 1237, 1, 1, 574, 2, 1241, 550, 118, 115, 229, + /* 70 */ 317, 480, 146, 480, 524, 118, 115, 229, 529, 1323, + /* 80 */ 417, 523, 142, 125, 126, 80, 1212, 1212, 1047, 1050, + /* 90 */ 1037, 1037, 123, 123, 124, 124, 124, 124, 118, 115, /* 100 */ 229, 327, 122, 122, 122, 122, 121, 121, 120, 120, /* 110 */ 120, 119, 116, 444, 284, 284, 284, 284, 442, 442, - /* 120 */ 442, 1567, 376, 1569, 1192, 375, 1163, 565, 1163, 565, - /* 130 */ 409, 1567, 537, 259, 226, 444, 101, 145, 449, 316, + /* 120 */ 442, 1561, 376, 1563, 1188, 375, 1159, 565, 1159, 565, + /* 130 */ 409, 1561, 537, 259, 226, 444, 101, 145, 449, 316, /* 140 */ 559, 240, 122, 122, 122, 122, 121, 121, 120, 120, - /* 150 */ 120, 119, 116, 444, 125, 126, 80, 1217, 1217, 1050, - /* 160 */ 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, 142, - /* 170 */ 294, 1192, 339, 448, 120, 120, 120, 119, 116, 444, - /* 180 */ 127, 1192, 1193, 1194, 148, 441, 440, 568, 119, 116, + /* 150 */ 120, 119, 116, 444, 125, 126, 80, 1212, 1212, 1047, + /* 160 */ 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, 142, + /* 170 */ 294, 1188, 339, 448, 120, 120, 120, 119, 116, 444, + /* 180 */ 127, 1188, 1189, 1188, 148, 441, 440, 568, 119, 116, /* 190 */ 444, 124, 124, 124, 124, 117, 122, 122, 122, 122, /* 200 */ 121, 121, 120, 120, 120, 119, 116, 444, 454, 113, /* 210 */ 13, 13, 546, 122, 122, 122, 122, 121, 121, 120, - /* 220 */ 120, 120, 119, 116, 444, 422, 316, 559, 1192, 1193, - /* 230 */ 1194, 149, 1224, 409, 1224, 124, 124, 124, 124, 122, + /* 220 */ 120, 120, 119, 116, 444, 422, 316, 559, 1188, 1189, + /* 230 */ 1188, 149, 1220, 409, 1220, 124, 124, 124, 124, 122, /* 240 */ 122, 122, 122, 121, 121, 120, 120, 120, 119, 116, - /* 250 */ 444, 465, 342, 1037, 1037, 1051, 1054, 125, 126, 80, - /* 260 */ 1217, 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, - /* 270 */ 124, 124, 1279, 522, 222, 1192, 568, 409, 224, 514, + /* 250 */ 444, 465, 342, 1034, 1034, 1048, 1051, 125, 126, 80, + /* 260 */ 1212, 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, + /* 270 */ 124, 124, 1275, 522, 222, 1188, 568, 409, 224, 514, /* 280 */ 175, 82, 83, 122, 122, 122, 122, 121, 121, 120, - /* 290 */ 120, 120, 119, 116, 444, 1007, 16, 16, 1192, 133, - /* 300 */ 133, 125, 126, 80, 1217, 1217, 1050, 1053, 1040, 1040, + /* 290 */ 120, 120, 119, 116, 444, 1005, 16, 16, 1188, 133, + /* 300 */ 133, 125, 126, 80, 1212, 1212, 1047, 1050, 1037, 1037, /* 310 */ 123, 123, 124, 124, 124, 124, 122, 122, 122, 122, - /* 320 */ 121, 121, 120, 120, 120, 119, 116, 444, 1041, 546, - /* 330 */ 1192, 373, 1192, 1193, 1194, 252, 1434, 399, 504, 501, - /* 340 */ 500, 111, 560, 566, 4, 926, 926, 433, 499, 340, - /* 350 */ 460, 328, 360, 394, 1237, 1192, 1193, 1194, 563, 568, + /* 320 */ 121, 121, 120, 120, 120, 119, 116, 444, 1038, 546, + /* 330 */ 1188, 373, 1188, 1189, 1188, 252, 1429, 399, 504, 501, + /* 340 */ 500, 111, 560, 566, 4, 924, 924, 433, 499, 340, + /* 350 */ 460, 328, 360, 394, 1233, 1188, 1189, 1188, 563, 568, /* 360 */ 122, 122, 122, 122, 121, 121, 120, 120, 120, 119, - /* 370 */ 116, 444, 284, 284, 369, 1580, 1607, 441, 440, 154, - /* 380 */ 409, 445, 71, 71, 1286, 565, 1221, 1192, 1193, 1194, - /* 390 */ 85, 1223, 271, 557, 543, 515, 1561, 568, 98, 1222, - /* 400 */ 6, 1278, 472, 142, 125, 126, 80, 1217, 1217, 1050, - /* 410 */ 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, 550, - /* 420 */ 13, 13, 1027, 507, 1224, 1192, 1224, 549, 109, 109, - /* 430 */ 222, 568, 1238, 175, 568, 427, 110, 197, 445, 570, - /* 440 */ 569, 430, 1552, 1017, 325, 551, 1192, 270, 287, 368, + /* 370 */ 116, 444, 284, 284, 369, 1574, 1600, 441, 440, 154, + /* 380 */ 409, 445, 71, 71, 1282, 565, 1217, 1188, 1189, 1188, + /* 390 */ 85, 1219, 271, 557, 543, 515, 1555, 568, 98, 1218, + /* 400 */ 6, 1274, 472, 142, 125, 126, 80, 1212, 1212, 1047, + /* 410 */ 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, 550, + /* 420 */ 13, 13, 1024, 507, 1220, 1188, 1220, 549, 109, 109, + /* 430 */ 222, 568, 1234, 175, 568, 427, 110, 197, 445, 569, + /* 440 */ 445, 430, 1546, 1014, 325, 551, 1188, 270, 287, 368, /* 450 */ 510, 363, 509, 257, 71, 71, 543, 71, 71, 359, - /* 460 */ 316, 559, 1613, 122, 122, 122, 122, 121, 121, 120, - /* 470 */ 120, 120, 119, 116, 444, 1017, 1017, 1019, 1020, 27, - /* 480 */ 284, 284, 1192, 1193, 1194, 1158, 568, 1612, 409, 901, - /* 490 */ 190, 550, 356, 565, 550, 937, 533, 517, 1158, 516, - /* 500 */ 413, 1158, 552, 1192, 1193, 1194, 568, 544, 1554, 51, - /* 510 */ 51, 214, 125, 126, 80, 1217, 1217, 1050, 1053, 1040, - /* 520 */ 1040, 123, 123, 124, 124, 124, 124, 1192, 474, 135, - /* 530 */ 135, 409, 284, 284, 1490, 505, 121, 121, 120, 120, - /* 540 */ 120, 119, 116, 444, 1007, 565, 518, 217, 541, 1561, - /* 550 */ 316, 559, 142, 6, 532, 125, 126, 80, 1217, 1217, - /* 560 */ 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, - /* 570 */ 1555, 122, 122, 122, 122, 121, 121, 120, 120, 120, - /* 580 */ 119, 116, 444, 485, 1192, 1193, 1194, 482, 281, 1267, - /* 590 */ 957, 252, 1192, 373, 504, 501, 500, 1192, 340, 571, - /* 600 */ 1192, 571, 409, 292, 499, 957, 876, 191, 480, 316, + /* 460 */ 316, 559, 1606, 122, 122, 122, 122, 121, 121, 120, + /* 470 */ 120, 120, 119, 116, 444, 1014, 1014, 1016, 1017, 27, + /* 480 */ 284, 284, 1188, 1189, 1188, 1154, 568, 1605, 409, 899, + /* 490 */ 190, 550, 356, 565, 550, 935, 533, 517, 1154, 516, + /* 500 */ 413, 1154, 552, 1188, 1189, 1188, 568, 544, 1548, 51, + /* 510 */ 51, 214, 125, 126, 80, 1212, 1212, 1047, 1050, 1037, + /* 520 */ 1037, 123, 123, 124, 124, 124, 124, 1188, 474, 135, + /* 530 */ 135, 409, 284, 284, 1484, 505, 121, 121, 120, 120, + /* 540 */ 120, 119, 116, 444, 1005, 565, 518, 217, 541, 1555, + /* 550 */ 316, 559, 142, 6, 532, 125, 126, 80, 1212, 1212, + /* 560 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, + /* 570 */ 1549, 122, 122, 122, 122, 121, 121, 120, 120, 120, + /* 580 */ 119, 116, 444, 485, 1188, 1189, 1188, 482, 281, 1263, + /* 590 */ 955, 252, 1188, 373, 504, 501, 500, 1188, 340, 570, + /* 600 */ 1188, 570, 409, 292, 499, 955, 874, 191, 480, 316, /* 610 */ 559, 384, 290, 380, 122, 122, 122, 122, 121, 121, - /* 620 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1217, - /* 630 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 640 */ 124, 409, 394, 1136, 1192, 869, 100, 284, 284, 1192, - /* 650 */ 1193, 1194, 373, 1093, 1192, 1193, 1194, 1192, 1193, 1194, - /* 660 */ 565, 455, 32, 373, 233, 125, 126, 80, 1217, 1217, - /* 670 */ 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, - /* 680 */ 1433, 959, 568, 228, 958, 122, 122, 122, 122, 121, - /* 690 */ 121, 120, 120, 120, 119, 116, 444, 1158, 228, 1192, - /* 700 */ 157, 1192, 1193, 1194, 1553, 13, 13, 301, 957, 1232, - /* 710 */ 1158, 153, 409, 1158, 373, 1583, 1176, 5, 369, 1580, - /* 720 */ 429, 1238, 3, 957, 122, 122, 122, 122, 121, 121, - /* 730 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1217, - /* 740 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 750 */ 124, 409, 208, 567, 1192, 1028, 1192, 1193, 1194, 1192, - /* 760 */ 388, 852, 155, 1552, 286, 402, 1098, 1098, 488, 568, - /* 770 */ 465, 342, 1319, 1319, 1552, 125, 126, 80, 1217, 1217, - /* 780 */ 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, + /* 620 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, + /* 630 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 640 */ 124, 409, 394, 1132, 1188, 867, 100, 284, 284, 1188, + /* 650 */ 1189, 1188, 373, 1089, 1188, 1189, 1188, 1188, 1189, 1188, + /* 660 */ 565, 455, 32, 373, 233, 125, 126, 80, 1212, 1212, + /* 670 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, + /* 680 */ 1428, 957, 568, 228, 956, 122, 122, 122, 122, 121, + /* 690 */ 121, 120, 120, 120, 119, 116, 444, 1154, 228, 1188, + /* 700 */ 157, 1188, 1189, 1188, 1547, 13, 13, 301, 955, 1228, + /* 710 */ 1154, 153, 409, 1154, 373, 1577, 1172, 5, 369, 1574, + /* 720 */ 429, 1234, 3, 955, 122, 122, 122, 122, 121, 121, + /* 730 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, + /* 740 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 750 */ 124, 409, 208, 567, 1188, 1025, 1188, 1189, 1188, 1188, + /* 760 */ 388, 850, 155, 1546, 286, 402, 1094, 1094, 488, 568, + /* 770 */ 465, 342, 1315, 1315, 1546, 125, 126, 80, 1212, 1212, + /* 780 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, /* 790 */ 129, 568, 13, 13, 374, 122, 122, 122, 122, 121, /* 800 */ 121, 120, 120, 120, 119, 116, 444, 302, 568, 453, - /* 810 */ 528, 1192, 1193, 1194, 13, 13, 1192, 1193, 1194, 1297, - /* 820 */ 463, 1267, 409, 1317, 1317, 1552, 1012, 453, 452, 200, - /* 830 */ 299, 71, 71, 1265, 122, 122, 122, 122, 121, 121, - /* 840 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1217, - /* 850 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 860 */ 124, 409, 227, 1073, 1158, 284, 284, 419, 312, 278, - /* 870 */ 278, 285, 285, 1419, 406, 405, 382, 1158, 565, 568, - /* 880 */ 1158, 1196, 565, 1600, 565, 125, 126, 80, 1217, 1217, - /* 890 */ 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, 124, - /* 900 */ 453, 1482, 13, 13, 1536, 122, 122, 122, 122, 121, + /* 810 */ 528, 1188, 1189, 1188, 13, 13, 1188, 1189, 1188, 1293, + /* 820 */ 463, 1263, 409, 1313, 1313, 1546, 1010, 453, 452, 200, + /* 830 */ 299, 71, 71, 1261, 122, 122, 122, 122, 121, 121, + /* 840 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, + /* 850 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 860 */ 124, 409, 227, 1069, 1154, 284, 284, 419, 312, 278, + /* 870 */ 278, 285, 285, 1415, 406, 405, 382, 1154, 565, 568, + /* 880 */ 1154, 1191, 565, 1594, 565, 125, 126, 80, 1212, 1212, + /* 890 */ 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, 124, + /* 900 */ 453, 1476, 13, 13, 1530, 122, 122, 122, 122, 121, /* 910 */ 121, 120, 120, 120, 119, 116, 444, 201, 568, 354, - /* 920 */ 1586, 575, 2, 1245, 840, 841, 842, 1562, 317, 1212, - /* 930 */ 146, 6, 409, 255, 254, 253, 206, 1327, 9, 1196, + /* 920 */ 1580, 574, 2, 1241, 838, 839, 840, 1556, 317, 1207, + /* 930 */ 146, 6, 409, 255, 254, 253, 206, 1323, 9, 1191, /* 940 */ 262, 71, 71, 424, 122, 122, 122, 122, 121, 121, - /* 950 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1217, - /* 960 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 970 */ 124, 568, 284, 284, 568, 1213, 409, 574, 313, 1245, - /* 980 */ 349, 1296, 352, 419, 317, 565, 146, 491, 525, 1643, - /* 990 */ 395, 371, 491, 1327, 70, 70, 1295, 71, 71, 240, - /* 1000 */ 1325, 104, 80, 1217, 1217, 1050, 1053, 1040, 1040, 123, + /* 950 */ 120, 120, 120, 119, 116, 444, 125, 126, 80, 1212, + /* 960 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 970 */ 124, 568, 284, 284, 568, 1208, 409, 573, 313, 1241, + /* 980 */ 349, 1292, 352, 419, 317, 565, 146, 491, 525, 1637, + /* 990 */ 395, 371, 491, 1323, 70, 70, 1291, 71, 71, 240, + /* 1000 */ 1321, 104, 80, 1212, 1212, 1047, 1050, 1037, 1037, 123, /* 1010 */ 123, 124, 124, 124, 124, 122, 122, 122, 122, 121, - /* 1020 */ 121, 120, 120, 120, 119, 116, 444, 1114, 284, 284, - /* 1030 */ 428, 448, 1525, 1213, 439, 284, 284, 1489, 1352, 311, - /* 1040 */ 474, 565, 1115, 971, 491, 491, 217, 1263, 565, 1538, - /* 1050 */ 568, 972, 207, 568, 1027, 240, 383, 1116, 519, 122, + /* 1020 */ 121, 120, 120, 120, 119, 116, 444, 1110, 284, 284, + /* 1030 */ 428, 448, 1519, 1208, 439, 284, 284, 1483, 1348, 311, + /* 1040 */ 474, 565, 1111, 969, 491, 491, 217, 1259, 565, 1532, + /* 1050 */ 568, 970, 207, 568, 1024, 240, 383, 1112, 519, 122, /* 1060 */ 122, 122, 122, 121, 121, 120, 120, 120, 119, 116, - /* 1070 */ 444, 1018, 107, 71, 71, 1017, 13, 13, 912, 568, - /* 1080 */ 1495, 568, 284, 284, 97, 526, 491, 448, 913, 1326, - /* 1090 */ 1322, 545, 409, 284, 284, 565, 151, 209, 1495, 1497, - /* 1100 */ 262, 450, 55, 55, 56, 56, 565, 1017, 1017, 1019, - /* 1110 */ 443, 332, 409, 527, 12, 295, 125, 126, 80, 1217, - /* 1120 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 1130 */ 124, 347, 409, 864, 1534, 1213, 125, 126, 80, 1217, - /* 1140 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 1150 */ 124, 1137, 1641, 474, 1641, 371, 125, 114, 80, 1217, - /* 1160 */ 1217, 1050, 1053, 1040, 1040, 123, 123, 124, 124, 124, - /* 1170 */ 124, 1495, 329, 474, 331, 122, 122, 122, 122, 121, - /* 1180 */ 121, 120, 120, 120, 119, 116, 444, 203, 1419, 568, - /* 1190 */ 1294, 864, 464, 1213, 436, 122, 122, 122, 122, 121, - /* 1200 */ 121, 120, 120, 120, 119, 116, 444, 553, 1137, 1642, - /* 1210 */ 539, 1642, 15, 15, 892, 122, 122, 122, 122, 121, + /* 1070 */ 444, 1015, 107, 71, 71, 1014, 13, 13, 910, 568, + /* 1080 */ 1489, 568, 284, 284, 97, 526, 491, 448, 911, 1322, + /* 1090 */ 1318, 545, 409, 284, 284, 565, 151, 209, 1489, 1491, + /* 1100 */ 262, 450, 55, 55, 56, 56, 565, 1014, 1014, 1016, + /* 1110 */ 443, 332, 409, 527, 12, 295, 125, 126, 80, 1212, + /* 1120 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 1130 */ 124, 347, 409, 862, 1528, 1208, 125, 126, 80, 1212, + /* 1140 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 1150 */ 124, 1133, 1635, 474, 1635, 371, 125, 114, 80, 1212, + /* 1160 */ 1212, 1047, 1050, 1037, 1037, 123, 123, 124, 124, 124, + /* 1170 */ 124, 1489, 329, 474, 331, 122, 122, 122, 122, 121, + /* 1180 */ 121, 120, 120, 120, 119, 116, 444, 203, 1415, 568, + /* 1190 */ 1290, 862, 464, 1208, 436, 122, 122, 122, 122, 121, + /* 1200 */ 121, 120, 120, 120, 119, 116, 444, 553, 1133, 1636, + /* 1210 */ 539, 1636, 15, 15, 890, 122, 122, 122, 122, 121, /* 1220 */ 121, 120, 120, 120, 119, 116, 444, 568, 298, 538, - /* 1230 */ 1135, 1419, 1559, 1560, 1331, 409, 6, 6, 1169, 1268, - /* 1240 */ 415, 320, 284, 284, 1419, 508, 565, 525, 300, 457, - /* 1250 */ 43, 43, 568, 893, 12, 565, 330, 478, 425, 407, - /* 1260 */ 126, 80, 1217, 1217, 1050, 1053, 1040, 1040, 123, 123, - /* 1270 */ 124, 124, 124, 124, 568, 57, 57, 288, 1192, 1419, - /* 1280 */ 496, 458, 392, 392, 391, 273, 389, 1135, 1558, 849, - /* 1290 */ 1169, 407, 6, 568, 321, 1158, 470, 44, 44, 1557, - /* 1300 */ 1114, 426, 234, 6, 323, 256, 540, 256, 1158, 431, - /* 1310 */ 568, 1158, 322, 17, 487, 1115, 58, 58, 122, 122, + /* 1230 */ 1131, 1415, 1553, 1554, 1327, 409, 6, 6, 1165, 1264, + /* 1240 */ 415, 320, 284, 284, 1415, 508, 565, 525, 300, 457, + /* 1250 */ 43, 43, 568, 891, 12, 565, 330, 478, 425, 407, + /* 1260 */ 126, 80, 1212, 1212, 1047, 1050, 1037, 1037, 123, 123, + /* 1270 */ 124, 124, 124, 124, 568, 57, 57, 288, 1188, 1415, + /* 1280 */ 496, 458, 392, 392, 391, 273, 389, 1131, 1552, 847, + /* 1290 */ 1165, 407, 6, 568, 321, 1154, 470, 44, 44, 1551, + /* 1300 */ 1110, 426, 234, 6, 323, 256, 540, 256, 1154, 431, + /* 1310 */ 568, 1154, 322, 17, 487, 1111, 58, 58, 122, 122, /* 1320 */ 122, 122, 121, 121, 120, 120, 120, 119, 116, 444, - /* 1330 */ 1116, 216, 481, 59, 59, 1192, 1193, 1194, 111, 560, + /* 1330 */ 1112, 216, 481, 59, 59, 1188, 1189, 1188, 111, 560, /* 1340 */ 324, 4, 236, 456, 526, 568, 237, 456, 568, 437, - /* 1350 */ 168, 556, 420, 141, 479, 563, 568, 293, 568, 1095, - /* 1360 */ 568, 293, 568, 1095, 531, 568, 872, 8, 60, 60, + /* 1350 */ 168, 556, 420, 141, 479, 563, 568, 293, 568, 1091, + /* 1360 */ 568, 293, 568, 1091, 531, 568, 870, 8, 60, 60, /* 1370 */ 235, 61, 61, 568, 414, 568, 414, 568, 445, 62, /* 1380 */ 62, 45, 45, 46, 46, 47, 47, 199, 49, 49, /* 1390 */ 557, 568, 359, 568, 100, 486, 50, 50, 63, 63, - /* 1400 */ 64, 64, 561, 415, 535, 410, 568, 1027, 568, 534, - /* 1410 */ 316, 559, 316, 559, 65, 65, 14, 14, 568, 1027, - /* 1420 */ 568, 512, 932, 872, 1018, 109, 109, 931, 1017, 66, - /* 1430 */ 66, 131, 131, 110, 451, 445, 570, 569, 416, 177, - /* 1440 */ 1017, 132, 132, 67, 67, 568, 467, 568, 932, 471, - /* 1450 */ 1364, 283, 226, 931, 315, 1363, 407, 568, 459, 407, - /* 1460 */ 1017, 1017, 1019, 239, 407, 86, 213, 1350, 52, 52, - /* 1470 */ 68, 68, 1017, 1017, 1019, 1020, 27, 1585, 1180, 447, - /* 1480 */ 69, 69, 288, 97, 108, 1541, 106, 392, 392, 391, - /* 1490 */ 273, 389, 568, 879, 849, 883, 568, 111, 560, 466, - /* 1500 */ 4, 568, 152, 30, 38, 568, 1132, 234, 396, 323, + /* 1400 */ 64, 64, 561, 415, 535, 410, 568, 1024, 568, 534, + /* 1410 */ 316, 559, 316, 559, 65, 65, 14, 14, 568, 1024, + /* 1420 */ 568, 512, 930, 870, 1015, 109, 109, 929, 1014, 66, + /* 1430 */ 66, 131, 131, 110, 451, 445, 569, 445, 416, 177, + /* 1440 */ 1014, 132, 132, 67, 67, 568, 467, 568, 930, 471, + /* 1450 */ 1360, 283, 226, 929, 315, 1359, 407, 568, 459, 407, + /* 1460 */ 1014, 1014, 1016, 239, 407, 86, 213, 1346, 52, 52, + /* 1470 */ 68, 68, 1014, 1014, 1016, 1017, 27, 1579, 1176, 447, + /* 1480 */ 69, 69, 288, 97, 108, 1535, 106, 392, 392, 391, + /* 1490 */ 273, 389, 568, 877, 847, 881, 568, 111, 560, 466, + /* 1500 */ 4, 568, 152, 30, 38, 568, 1128, 234, 396, 323, /* 1510 */ 111, 560, 527, 4, 563, 53, 53, 322, 568, 163, /* 1520 */ 163, 568, 337, 468, 164, 164, 333, 563, 76, 76, - /* 1530 */ 568, 289, 1514, 568, 31, 1513, 568, 445, 338, 483, - /* 1540 */ 100, 54, 54, 344, 72, 72, 296, 236, 1080, 557, - /* 1550 */ 445, 879, 1360, 134, 134, 168, 73, 73, 141, 161, - /* 1560 */ 161, 1574, 557, 535, 568, 319, 568, 348, 536, 1009, - /* 1570 */ 473, 261, 261, 891, 890, 235, 535, 568, 1027, 568, + /* 1530 */ 568, 289, 1508, 568, 31, 1507, 568, 445, 338, 483, + /* 1540 */ 100, 54, 54, 344, 72, 72, 296, 236, 1076, 557, + /* 1550 */ 445, 877, 1356, 134, 134, 168, 73, 73, 141, 161, + /* 1560 */ 161, 1568, 557, 535, 568, 319, 568, 348, 536, 1007, + /* 1570 */ 473, 261, 261, 889, 888, 235, 535, 568, 1024, 568, /* 1580 */ 475, 534, 261, 367, 109, 109, 521, 136, 136, 130, - /* 1590 */ 130, 1027, 110, 366, 445, 570, 569, 109, 109, 1017, - /* 1600 */ 162, 162, 156, 156, 568, 110, 1080, 445, 570, 569, - /* 1610 */ 410, 351, 1017, 568, 353, 316, 559, 568, 343, 568, - /* 1620 */ 100, 497, 357, 258, 100, 898, 899, 140, 140, 355, - /* 1630 */ 1310, 1017, 1017, 1019, 1020, 27, 139, 139, 362, 451, - /* 1640 */ 137, 137, 138, 138, 1017, 1017, 1019, 1020, 27, 1180, - /* 1650 */ 447, 568, 372, 288, 111, 560, 1021, 4, 392, 392, - /* 1660 */ 391, 273, 389, 568, 1141, 849, 568, 1076, 568, 258, - /* 1670 */ 492, 563, 568, 211, 75, 75, 555, 962, 234, 261, - /* 1680 */ 323, 111, 560, 929, 4, 113, 77, 77, 322, 74, - /* 1690 */ 74, 42, 42, 1373, 445, 48, 48, 1418, 563, 974, - /* 1700 */ 975, 1092, 1091, 1092, 1091, 862, 557, 150, 930, 1346, - /* 1710 */ 113, 1358, 554, 1424, 1021, 1275, 1266, 1254, 236, 1253, - /* 1720 */ 1255, 445, 1593, 1343, 308, 276, 168, 309, 11, 141, - /* 1730 */ 393, 310, 232, 557, 1405, 1027, 335, 291, 1400, 219, - /* 1740 */ 336, 109, 109, 936, 297, 1410, 235, 341, 477, 110, - /* 1750 */ 502, 445, 570, 569, 1393, 1409, 1017, 400, 1293, 365, - /* 1760 */ 223, 1486, 1027, 1485, 1355, 1356, 1354, 1353, 109, 109, - /* 1770 */ 204, 1596, 1232, 558, 265, 218, 110, 205, 445, 570, - /* 1780 */ 569, 410, 387, 1017, 1533, 179, 316, 559, 1017, 1017, - /* 1790 */ 1019, 1020, 27, 230, 1531, 1229, 79, 560, 85, 4, - /* 1800 */ 418, 215, 548, 81, 84, 188, 1406, 173, 181, 461, - /* 1810 */ 451, 35, 462, 563, 183, 1017, 1017, 1019, 1020, 27, - /* 1820 */ 184, 1491, 185, 186, 495, 242, 98, 398, 1412, 36, - /* 1830 */ 1411, 484, 91, 469, 401, 1414, 445, 192, 1480, 246, - /* 1840 */ 1502, 490, 346, 277, 248, 196, 493, 511, 557, 350, - /* 1850 */ 1256, 249, 250, 403, 1313, 1312, 111, 560, 432, 4, - /* 1860 */ 1311, 1304, 93, 1611, 883, 1610, 224, 404, 434, 520, - /* 1870 */ 263, 435, 1579, 563, 1283, 1282, 364, 1027, 306, 1281, - /* 1880 */ 264, 1609, 1565, 109, 109, 370, 1303, 307, 1564, 438, - /* 1890 */ 128, 110, 1378, 445, 570, 569, 445, 546, 1017, 10, - /* 1900 */ 1466, 105, 381, 1377, 34, 572, 99, 1336, 557, 314, - /* 1910 */ 1186, 530, 272, 274, 379, 210, 1335, 547, 385, 386, - /* 1920 */ 275, 573, 1251, 1246, 411, 412, 1518, 165, 178, 1519, - /* 1930 */ 1017, 1017, 1019, 1020, 27, 1517, 1516, 1027, 78, 147, - /* 1940 */ 166, 220, 221, 109, 109, 836, 304, 167, 446, 212, - /* 1950 */ 318, 110, 231, 445, 570, 569, 144, 1090, 1017, 1088, - /* 1960 */ 326, 180, 169, 1212, 182, 334, 238, 915, 241, 1104, + /* 1590 */ 130, 1024, 110, 366, 445, 569, 445, 109, 109, 1014, + /* 1600 */ 162, 162, 156, 156, 568, 110, 1076, 445, 569, 445, + /* 1610 */ 410, 351, 1014, 568, 353, 316, 559, 568, 343, 568, + /* 1620 */ 100, 497, 357, 258, 100, 896, 897, 140, 140, 355, + /* 1630 */ 1306, 1014, 1014, 1016, 1017, 27, 139, 139, 362, 451, + /* 1640 */ 137, 137, 138, 138, 1014, 1014, 1016, 1017, 27, 1176, + /* 1650 */ 447, 568, 372, 288, 111, 560, 1018, 4, 392, 392, + /* 1660 */ 391, 273, 389, 568, 1137, 847, 568, 1072, 568, 258, + /* 1670 */ 492, 563, 568, 211, 75, 75, 555, 960, 234, 261, + /* 1680 */ 323, 111, 560, 927, 4, 113, 77, 77, 322, 74, + /* 1690 */ 74, 42, 42, 1369, 445, 48, 48, 1414, 563, 972, + /* 1700 */ 973, 1088, 1087, 1088, 1087, 860, 557, 150, 928, 1342, + /* 1710 */ 113, 1354, 554, 1419, 1018, 1271, 1262, 1250, 236, 1249, + /* 1720 */ 1251, 445, 1587, 1339, 308, 276, 168, 309, 11, 141, + /* 1730 */ 393, 310, 232, 557, 1401, 1024, 335, 291, 1396, 219, + /* 1740 */ 336, 109, 109, 934, 297, 1406, 235, 341, 477, 110, + /* 1750 */ 502, 445, 569, 445, 1389, 1405, 1014, 400, 1289, 365, + /* 1760 */ 223, 1480, 1024, 1479, 1351, 1352, 1350, 1349, 109, 109, + /* 1770 */ 204, 1590, 1228, 558, 265, 218, 110, 205, 445, 569, + /* 1780 */ 445, 410, 387, 1014, 1527, 179, 316, 559, 1014, 1014, + /* 1790 */ 1016, 1017, 27, 230, 1525, 1225, 79, 560, 85, 4, + /* 1800 */ 418, 215, 548, 81, 84, 188, 1402, 173, 181, 461, + /* 1810 */ 451, 35, 462, 563, 183, 1014, 1014, 1016, 1017, 27, + /* 1820 */ 184, 1485, 185, 186, 495, 242, 98, 398, 1408, 36, + /* 1830 */ 1407, 484, 91, 469, 401, 1410, 445, 192, 1474, 246, + /* 1840 */ 1496, 490, 346, 277, 248, 196, 493, 511, 557, 350, + /* 1850 */ 1252, 249, 250, 403, 1309, 1308, 111, 560, 432, 4, + /* 1860 */ 1307, 1300, 93, 1604, 881, 1603, 224, 404, 434, 520, + /* 1870 */ 263, 435, 1573, 563, 1279, 1278, 364, 1024, 306, 1277, + /* 1880 */ 264, 1602, 1559, 109, 109, 370, 1299, 307, 1558, 438, + /* 1890 */ 128, 110, 1374, 445, 569, 445, 445, 546, 1014, 10, + /* 1900 */ 1461, 105, 381, 1373, 34, 571, 99, 1332, 557, 314, + /* 1910 */ 1182, 530, 272, 274, 379, 210, 1331, 547, 385, 386, + /* 1920 */ 275, 572, 1247, 1242, 411, 412, 1512, 165, 178, 1513, + /* 1930 */ 1014, 1014, 1016, 1017, 27, 1511, 1510, 1024, 78, 147, + /* 1940 */ 166, 220, 221, 109, 109, 834, 304, 167, 446, 212, + /* 1950 */ 318, 110, 231, 445, 569, 445, 144, 1086, 1014, 1084, + /* 1960 */ 326, 180, 169, 1207, 182, 334, 238, 913, 241, 1100, /* 1970 */ 187, 170, 171, 421, 87, 88, 423, 189, 89, 90, - /* 1980 */ 172, 1107, 243, 1103, 244, 158, 18, 245, 345, 247, - /* 1990 */ 1017, 1017, 1019, 1020, 27, 261, 1096, 193, 1226, 489, - /* 2000 */ 194, 37, 366, 851, 494, 251, 195, 506, 92, 19, - /* 2010 */ 498, 358, 20, 503, 881, 361, 94, 894, 305, 159, - /* 2020 */ 513, 39, 95, 1174, 160, 1056, 966, 1143, 96, 174, - /* 2030 */ 1142, 225, 280, 282, 198, 960, 113, 1164, 1160, 260, - /* 2040 */ 21, 22, 23, 1162, 1168, 1167, 1148, 24, 33, 25, - /* 2050 */ 202, 542, 26, 100, 1071, 102, 1057, 103, 7, 1055, - /* 2060 */ 1059, 1113, 1060, 1112, 266, 267, 28, 40, 390, 1022, - /* 2070 */ 863, 112, 29, 564, 1182, 1181, 268, 176, 143, 925, - /* 2080 */ 1242, 1242, 1242, 1242, 1242, 1242, 1242, 1242, 1242, 1242, - /* 2090 */ 1242, 1242, 1242, 1242, 269, 1602, 1242, 1601, + /* 1980 */ 172, 1103, 243, 1099, 244, 158, 18, 245, 345, 247, + /* 1990 */ 1014, 1014, 1016, 1017, 27, 261, 1092, 193, 1222, 489, + /* 2000 */ 194, 37, 366, 849, 494, 251, 195, 506, 92, 19, + /* 2010 */ 498, 358, 20, 503, 879, 361, 94, 892, 305, 159, + /* 2020 */ 513, 39, 95, 1170, 160, 1053, 964, 1139, 96, 174, + /* 2030 */ 1138, 225, 280, 282, 198, 958, 113, 1160, 1156, 260, + /* 2040 */ 21, 22, 23, 1158, 1164, 1163, 1144, 24, 33, 25, + /* 2050 */ 202, 542, 26, 100, 1067, 102, 1054, 103, 7, 1052, + /* 2060 */ 1056, 1109, 1057, 1108, 266, 267, 28, 40, 390, 1019, + /* 2070 */ 861, 112, 29, 564, 1178, 1177, 268, 176, 143, 923, + /* 2080 */ 1238, 1238, 1238, 1238, 1238, 1238, 1238, 1238, 1238, 1238, + /* 2090 */ 1238, 1238, 1238, 1238, 269, 1595, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 193, 193, 193, 274, 275, 276, 193, 274, 275, 276, @@ -167786,7 +168729,7 @@ static const YYCODETYPE yy_lookahead[] = { /* 2060 */ 23, 23, 11, 23, 25, 22, 22, 22, 15, 23, /* 2070 */ 23, 22, 22, 25, 1, 1, 141, 25, 23, 135, /* 2080 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2090 */ 319, 319, 319, 319, 141, 141, 319, 141, 319, 319, + /* 2090 */ 319, 319, 319, 319, 141, 141, 319, 319, 319, 319, /* 2100 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2110 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2120 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, @@ -167805,9 +168748,9 @@ static const YYCODETYPE yy_lookahead[] = { /* 2250 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2260 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, /* 2270 */ 319, 319, 319, 319, 319, 319, 319, 319, 319, 319, - /* 2280 */ 319, 319, 319, + /* 2280 */ 319, }; -#define YY_SHIFT_COUNT (575) +#define YY_SHIFT_COUNT (574) #define YY_SHIFT_MIN (0) #define YY_SHIFT_MAX (2074) static const unsigned short int yy_shift_ofst[] = { @@ -167827,12 +168770,12 @@ static const unsigned short int yy_shift_ofst[] = { /* 130 */ 137, 181, 181, 181, 181, 181, 181, 181, 94, 430, /* 140 */ 66, 65, 112, 366, 533, 533, 740, 1261, 533, 533, /* 150 */ 79, 79, 533, 412, 412, 412, 77, 412, 123, 113, - /* 160 */ 113, 22, 22, 2098, 2098, 328, 328, 328, 239, 468, + /* 160 */ 113, 22, 22, 2096, 2096, 328, 328, 328, 239, 468, /* 170 */ 468, 468, 468, 1015, 1015, 409, 366, 1129, 1186, 533, /* 180 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, /* 190 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 969, /* 200 */ 621, 621, 533, 642, 788, 788, 1228, 1228, 822, 822, - /* 210 */ 67, 1274, 2098, 2098, 2098, 2098, 2098, 2098, 2098, 1307, + /* 210 */ 67, 1274, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 1307, /* 220 */ 954, 954, 585, 472, 640, 387, 695, 538, 541, 700, /* 230 */ 533, 533, 533, 533, 533, 533, 533, 533, 533, 533, /* 240 */ 222, 533, 533, 533, 533, 533, 533, 533, 533, 533, @@ -167850,8 +168793,8 @@ static const unsigned short int yy_shift_ofst[] = { /* 360 */ 1840, 1840, 1823, 1732, 1738, 1732, 1794, 1732, 1732, 1701, /* 370 */ 1844, 1758, 1758, 1823, 1633, 1789, 1789, 1807, 1807, 1742, /* 380 */ 1752, 1877, 1633, 1743, 1742, 1759, 1765, 1677, 1879, 1897, - /* 390 */ 1897, 1914, 1914, 1914, 2098, 2098, 2098, 2098, 2098, 2098, - /* 400 */ 2098, 2098, 2098, 2098, 2098, 2098, 2098, 2098, 2098, 207, + /* 390 */ 1897, 1914, 1914, 1914, 2096, 2096, 2096, 2096, 2096, 2096, + /* 400 */ 2096, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 207, /* 410 */ 1095, 331, 620, 903, 806, 1074, 1483, 1432, 1481, 1322, /* 420 */ 1370, 1394, 1515, 1291, 1546, 1547, 1557, 1595, 1598, 1599, /* 430 */ 1434, 1453, 1618, 1462, 1567, 1489, 1644, 1654, 1616, 1660, @@ -167868,7 +168811,7 @@ static const unsigned short int yy_shift_ofst[] = { /* 540 */ 2015, 2023, 2026, 2027, 2025, 2028, 2018, 1913, 1915, 2031, /* 550 */ 2011, 2033, 2036, 2037, 2038, 2039, 2040, 2043, 2051, 2044, /* 560 */ 2045, 2046, 2047, 2049, 2050, 2048, 1944, 1935, 1953, 1954, - /* 570 */ 1956, 2052, 2055, 2053, 2073, 2074, + /* 570 */ 2052, 2055, 2053, 2073, 2074, }; #define YY_REDUCE_COUNT (408) #define YY_REDUCE_MIN (-271) @@ -167917,64 +168860,64 @@ static const short yy_reduce_ofst[] = { /* 400 */ 1722, 1723, 1733, 1717, 1724, 1727, 1728, 1725, 1740, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1647, 1647, 1647, 1475, 1240, 1351, 1240, 1240, 1240, 1475, - /* 10 */ 1475, 1475, 1240, 1381, 1381, 1528, 1273, 1240, 1240, 1240, - /* 20 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1474, 1240, 1240, - /* 30 */ 1240, 1240, 1563, 1563, 1240, 1240, 1240, 1240, 1240, 1240, - /* 40 */ 1240, 1240, 1390, 1240, 1397, 1240, 1240, 1240, 1240, 1240, - /* 50 */ 1476, 1477, 1240, 1240, 1240, 1527, 1529, 1492, 1404, 1403, - /* 60 */ 1402, 1401, 1510, 1369, 1395, 1388, 1392, 1470, 1471, 1469, - /* 70 */ 1473, 1477, 1476, 1240, 1391, 1438, 1454, 1437, 1240, 1240, - /* 80 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 90 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 100 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 110 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 120 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 130 */ 1446, 1453, 1452, 1451, 1460, 1450, 1447, 1440, 1439, 1441, - /* 140 */ 1442, 1240, 1240, 1264, 1240, 1240, 1261, 1315, 1240, 1240, - /* 150 */ 1240, 1240, 1240, 1547, 1546, 1240, 1443, 1240, 1273, 1432, - /* 160 */ 1431, 1457, 1444, 1456, 1455, 1535, 1599, 1598, 1493, 1240, - /* 170 */ 1240, 1240, 1240, 1240, 1240, 1563, 1240, 1240, 1240, 1240, - /* 180 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 190 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1371, - /* 200 */ 1563, 1563, 1240, 1273, 1563, 1563, 1372, 1372, 1269, 1269, - /* 210 */ 1375, 1240, 1542, 1342, 1342, 1342, 1342, 1351, 1342, 1240, - /* 220 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 230 */ 1240, 1240, 1240, 1240, 1532, 1530, 1240, 1240, 1240, 1240, - /* 240 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 250 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 260 */ 1240, 1240, 1240, 1347, 1240, 1240, 1240, 1240, 1240, 1240, - /* 270 */ 1240, 1240, 1240, 1240, 1240, 1592, 1240, 1505, 1329, 1347, - /* 280 */ 1347, 1347, 1347, 1349, 1330, 1328, 1341, 1274, 1247, 1639, - /* 290 */ 1407, 1396, 1348, 1396, 1636, 1394, 1407, 1407, 1394, 1407, - /* 300 */ 1348, 1636, 1290, 1615, 1285, 1381, 1381, 1381, 1371, 1371, - /* 310 */ 1371, 1371, 1375, 1375, 1472, 1348, 1341, 1240, 1639, 1639, - /* 320 */ 1357, 1357, 1638, 1638, 1357, 1493, 1623, 1416, 1318, 1324, - /* 330 */ 1324, 1324, 1324, 1357, 1258, 1394, 1623, 1623, 1394, 1416, - /* 340 */ 1318, 1394, 1318, 1394, 1357, 1258, 1509, 1633, 1357, 1258, - /* 350 */ 1483, 1357, 1258, 1357, 1258, 1483, 1316, 1316, 1316, 1305, - /* 360 */ 1240, 1240, 1483, 1316, 1290, 1316, 1305, 1316, 1316, 1581, - /* 370 */ 1240, 1487, 1487, 1483, 1357, 1573, 1573, 1384, 1384, 1389, - /* 380 */ 1375, 1478, 1357, 1240, 1389, 1387, 1385, 1394, 1308, 1595, - /* 390 */ 1595, 1591, 1591, 1591, 1644, 1644, 1542, 1608, 1273, 1273, - /* 400 */ 1273, 1273, 1608, 1292, 1292, 1274, 1274, 1273, 1608, 1240, - /* 410 */ 1240, 1240, 1240, 1240, 1240, 1603, 1240, 1537, 1494, 1361, - /* 420 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 430 */ 1240, 1240, 1240, 1240, 1548, 1240, 1240, 1240, 1240, 1240, - /* 440 */ 1240, 1240, 1240, 1240, 1240, 1421, 1240, 1243, 1539, 1240, - /* 450 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1398, 1399, 1362, - /* 460 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1413, 1240, 1240, - /* 470 */ 1240, 1408, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 480 */ 1635, 1240, 1240, 1240, 1240, 1240, 1240, 1508, 1507, 1240, - /* 490 */ 1240, 1359, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 500 */ 1240, 1240, 1240, 1240, 1240, 1288, 1240, 1240, 1240, 1240, - /* 510 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 520 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1386, - /* 530 */ 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 540 */ 1240, 1240, 1240, 1240, 1578, 1376, 1240, 1240, 1240, 1240, - /* 550 */ 1626, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, - /* 560 */ 1240, 1240, 1240, 1240, 1240, 1619, 1332, 1423, 1240, 1422, - /* 570 */ 1426, 1262, 1240, 1252, 1240, 1240, + /* 0 */ 1641, 1641, 1641, 1469, 1236, 1347, 1236, 1236, 1236, 1469, + /* 10 */ 1469, 1469, 1236, 1377, 1377, 1522, 1269, 1236, 1236, 1236, + /* 20 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1468, 1236, 1236, + /* 30 */ 1236, 1236, 1557, 1557, 1236, 1236, 1236, 1236, 1236, 1236, + /* 40 */ 1236, 1236, 1386, 1236, 1393, 1236, 1236, 1236, 1236, 1236, + /* 50 */ 1470, 1471, 1236, 1236, 1236, 1521, 1523, 1486, 1400, 1399, + /* 60 */ 1398, 1397, 1504, 1365, 1391, 1384, 1388, 1465, 1466, 1464, + /* 70 */ 1619, 1471, 1470, 1236, 1387, 1433, 1449, 1432, 1236, 1236, + /* 80 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 90 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 100 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 110 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 120 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 130 */ 1441, 1448, 1447, 1446, 1455, 1445, 1442, 1435, 1434, 1436, + /* 140 */ 1437, 1236, 1236, 1260, 1236, 1236, 1257, 1311, 1236, 1236, + /* 150 */ 1236, 1236, 1236, 1541, 1540, 1236, 1438, 1236, 1269, 1427, + /* 160 */ 1426, 1452, 1439, 1451, 1450, 1529, 1593, 1592, 1487, 1236, + /* 170 */ 1236, 1236, 1236, 1236, 1236, 1557, 1236, 1236, 1236, 1236, + /* 180 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 190 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1367, + /* 200 */ 1557, 1557, 1236, 1269, 1557, 1557, 1368, 1368, 1265, 1265, + /* 210 */ 1371, 1236, 1536, 1338, 1338, 1338, 1338, 1347, 1338, 1236, + /* 220 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 230 */ 1236, 1236, 1236, 1236, 1526, 1524, 1236, 1236, 1236, 1236, + /* 240 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 250 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 260 */ 1236, 1236, 1236, 1343, 1236, 1236, 1236, 1236, 1236, 1236, + /* 270 */ 1236, 1236, 1236, 1236, 1236, 1586, 1236, 1499, 1325, 1343, + /* 280 */ 1343, 1343, 1343, 1345, 1326, 1324, 1337, 1270, 1243, 1633, + /* 290 */ 1403, 1392, 1344, 1392, 1630, 1390, 1403, 1403, 1390, 1403, + /* 300 */ 1344, 1630, 1286, 1608, 1281, 1377, 1377, 1377, 1367, 1367, + /* 310 */ 1367, 1367, 1371, 1371, 1467, 1344, 1337, 1236, 1633, 1633, + /* 320 */ 1353, 1353, 1632, 1632, 1353, 1487, 1616, 1412, 1314, 1320, + /* 330 */ 1320, 1320, 1320, 1353, 1254, 1390, 1616, 1616, 1390, 1412, + /* 340 */ 1314, 1390, 1314, 1390, 1353, 1254, 1503, 1627, 1353, 1254, + /* 350 */ 1477, 1353, 1254, 1353, 1254, 1477, 1312, 1312, 1312, 1301, + /* 360 */ 1236, 1236, 1477, 1312, 1286, 1312, 1301, 1312, 1312, 1575, + /* 370 */ 1236, 1481, 1481, 1477, 1353, 1567, 1567, 1380, 1380, 1385, + /* 380 */ 1371, 1472, 1353, 1236, 1385, 1383, 1381, 1390, 1304, 1589, + /* 390 */ 1589, 1585, 1585, 1585, 1638, 1638, 1536, 1601, 1269, 1269, + /* 400 */ 1269, 1269, 1601, 1288, 1288, 1270, 1270, 1269, 1601, 1236, + /* 410 */ 1236, 1236, 1236, 1236, 1236, 1596, 1236, 1531, 1488, 1357, + /* 420 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 430 */ 1236, 1236, 1236, 1236, 1542, 1236, 1236, 1236, 1236, 1236, + /* 440 */ 1236, 1236, 1236, 1236, 1236, 1417, 1236, 1239, 1533, 1236, + /* 450 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1394, 1395, 1358, + /* 460 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1409, 1236, 1236, + /* 470 */ 1236, 1404, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 480 */ 1629, 1236, 1236, 1236, 1236, 1236, 1236, 1502, 1501, 1236, + /* 490 */ 1236, 1355, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 500 */ 1236, 1236, 1236, 1236, 1236, 1284, 1236, 1236, 1236, 1236, + /* 510 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 520 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1382, + /* 530 */ 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 540 */ 1236, 1236, 1236, 1236, 1572, 1372, 1236, 1236, 1236, 1236, + /* 550 */ 1620, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, 1236, + /* 560 */ 1236, 1236, 1236, 1236, 1236, 1612, 1328, 1418, 1236, 1421, + /* 570 */ 1258, 1236, 1248, 1236, 1236, }; /********** End of lemon-generated parsing tables *****************************/ @@ -168771,233 +169714,231 @@ static const char *const yyRuleName[] = { /* 175 */ "idlist ::= idlist COMMA nm", /* 176 */ "idlist ::= nm", /* 177 */ "expr ::= LP expr RP", - /* 178 */ "expr ::= ID|INDEXED", - /* 179 */ "expr ::= JOIN_KW", - /* 180 */ "expr ::= nm DOT nm", - /* 181 */ "expr ::= nm DOT nm DOT nm", - /* 182 */ "term ::= NULL|FLOAT|BLOB", - /* 183 */ "term ::= STRING", - /* 184 */ "term ::= INTEGER", - /* 185 */ "expr ::= VARIABLE", - /* 186 */ "expr ::= expr COLLATE ID|STRING", - /* 187 */ "expr ::= CAST LP expr AS typetoken RP", - /* 188 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 189 */ "expr ::= ID|INDEXED LP STAR RP", - /* 190 */ "expr ::= ID|INDEXED LP distinct exprlist RP filter_over", - /* 191 */ "expr ::= ID|INDEXED LP STAR RP filter_over", - /* 192 */ "term ::= CTIME_KW", - /* 193 */ "expr ::= LP nexprlist COMMA expr RP", - /* 194 */ "expr ::= expr AND expr", - /* 195 */ "expr ::= expr OR expr", - /* 196 */ "expr ::= expr LT|GT|GE|LE expr", - /* 197 */ "expr ::= expr EQ|NE expr", - /* 198 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 199 */ "expr ::= expr PLUS|MINUS expr", - /* 200 */ "expr ::= expr STAR|SLASH|REM expr", - /* 201 */ "expr ::= expr CONCAT expr", - /* 202 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 203 */ "expr ::= expr likeop expr", - /* 204 */ "expr ::= expr likeop expr ESCAPE expr", - /* 205 */ "expr ::= expr ISNULL|NOTNULL", - /* 206 */ "expr ::= expr NOT NULL", - /* 207 */ "expr ::= expr IS expr", - /* 208 */ "expr ::= expr IS NOT expr", - /* 209 */ "expr ::= expr IS NOT DISTINCT FROM expr", - /* 210 */ "expr ::= expr IS DISTINCT FROM expr", - /* 211 */ "expr ::= NOT expr", - /* 212 */ "expr ::= BITNOT expr", - /* 213 */ "expr ::= PLUS|MINUS expr", - /* 214 */ "expr ::= expr PTR expr", - /* 215 */ "between_op ::= BETWEEN", - /* 216 */ "between_op ::= NOT BETWEEN", - /* 217 */ "expr ::= expr between_op expr AND expr", - /* 218 */ "in_op ::= IN", - /* 219 */ "in_op ::= NOT IN", - /* 220 */ "expr ::= expr in_op LP exprlist RP", - /* 221 */ "expr ::= LP select RP", - /* 222 */ "expr ::= expr in_op LP select RP", - /* 223 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 224 */ "expr ::= EXISTS LP select RP", - /* 225 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 226 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 227 */ "case_exprlist ::= WHEN expr THEN expr", - /* 228 */ "case_else ::= ELSE expr", - /* 229 */ "case_else ::=", - /* 230 */ "case_operand ::= expr", - /* 231 */ "case_operand ::=", - /* 232 */ "exprlist ::=", - /* 233 */ "nexprlist ::= nexprlist COMMA expr", - /* 234 */ "nexprlist ::= expr", - /* 235 */ "paren_exprlist ::=", - /* 236 */ "paren_exprlist ::= LP exprlist RP", - /* 237 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 238 */ "uniqueflag ::= UNIQUE", - /* 239 */ "uniqueflag ::=", - /* 240 */ "eidlist_opt ::=", - /* 241 */ "eidlist_opt ::= LP eidlist RP", - /* 242 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 243 */ "eidlist ::= nm collate sortorder", - /* 244 */ "collate ::=", - /* 245 */ "collate ::= COLLATE ID|STRING", - /* 246 */ "cmd ::= DROP INDEX ifexists fullname", - /* 247 */ "cmd ::= VACUUM vinto", - /* 248 */ "cmd ::= VACUUM nm vinto", - /* 249 */ "vinto ::= INTO expr", - /* 250 */ "vinto ::=", - /* 251 */ "cmd ::= PRAGMA nm dbnm", - /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 256 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 257 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 258 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 259 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 260 */ "trigger_time ::= BEFORE|AFTER", - /* 261 */ "trigger_time ::= INSTEAD OF", - /* 262 */ "trigger_time ::=", - /* 263 */ "trigger_event ::= DELETE|INSERT", - /* 264 */ "trigger_event ::= UPDATE", - /* 265 */ "trigger_event ::= UPDATE OF idlist", - /* 266 */ "when_clause ::=", - /* 267 */ "when_clause ::= WHEN expr", - /* 268 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 269 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 270 */ "trnm ::= nm DOT nm", - /* 271 */ "tridxby ::= INDEXED BY nm", - /* 272 */ "tridxby ::= NOT INDEXED", - /* 273 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", - /* 274 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 275 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 276 */ "trigger_cmd ::= scanpt select scanpt", - /* 277 */ "expr ::= RAISE LP IGNORE RP", - /* 278 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 279 */ "raisetype ::= ROLLBACK", - /* 280 */ "raisetype ::= ABORT", - /* 281 */ "raisetype ::= FAIL", - /* 282 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 283 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 284 */ "cmd ::= DETACH database_kw_opt expr", - /* 285 */ "key_opt ::=", - /* 286 */ "key_opt ::= KEY expr", - /* 287 */ "cmd ::= REINDEX", - /* 288 */ "cmd ::= REINDEX nm dbnm", - /* 289 */ "cmd ::= ANALYZE", - /* 290 */ "cmd ::= ANALYZE nm dbnm", - /* 291 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 292 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 293 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", - /* 294 */ "add_column_fullname ::= fullname", - /* 295 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", - /* 296 */ "cmd ::= create_vtab", - /* 297 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 298 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 299 */ "vtabarg ::=", - /* 300 */ "vtabargtoken ::= ANY", - /* 301 */ "vtabargtoken ::= lp anylist RP", - /* 302 */ "lp ::= LP", - /* 303 */ "with ::= WITH wqlist", - /* 304 */ "with ::= WITH RECURSIVE wqlist", - /* 305 */ "wqas ::= AS", - /* 306 */ "wqas ::= AS MATERIALIZED", - /* 307 */ "wqas ::= AS NOT MATERIALIZED", - /* 308 */ "wqitem ::= nm eidlist_opt wqas LP select RP", - /* 309 */ "wqlist ::= wqitem", - /* 310 */ "wqlist ::= wqlist COMMA wqitem", - /* 311 */ "windowdefn_list ::= windowdefn", - /* 312 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", - /* 313 */ "windowdefn ::= nm AS LP window RP", - /* 314 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", - /* 315 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", - /* 316 */ "window ::= ORDER BY sortlist frame_opt", - /* 317 */ "window ::= nm ORDER BY sortlist frame_opt", - /* 318 */ "window ::= frame_opt", - /* 319 */ "window ::= nm frame_opt", - /* 320 */ "frame_opt ::=", - /* 321 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", - /* 322 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", - /* 323 */ "range_or_rows ::= RANGE|ROWS|GROUPS", - /* 324 */ "frame_bound_s ::= frame_bound", - /* 325 */ "frame_bound_s ::= UNBOUNDED PRECEDING", - /* 326 */ "frame_bound_e ::= frame_bound", - /* 327 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", - /* 328 */ "frame_bound ::= expr PRECEDING|FOLLOWING", - /* 329 */ "frame_bound ::= CURRENT ROW", - /* 330 */ "frame_exclude_opt ::=", - /* 331 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", - /* 332 */ "frame_exclude ::= NO OTHERS", - /* 333 */ "frame_exclude ::= CURRENT ROW", - /* 334 */ "frame_exclude ::= GROUP|TIES", - /* 335 */ "window_clause ::= WINDOW windowdefn_list", - /* 336 */ "filter_over ::= filter_clause over_clause", - /* 337 */ "filter_over ::= over_clause", - /* 338 */ "filter_over ::= filter_clause", - /* 339 */ "over_clause ::= OVER LP window RP", - /* 340 */ "over_clause ::= OVER nm", - /* 341 */ "filter_clause ::= FILTER LP WHERE expr RP", - /* 342 */ "input ::= cmdlist", - /* 343 */ "cmdlist ::= cmdlist ecmd", - /* 344 */ "cmdlist ::= ecmd", - /* 345 */ "ecmd ::= SEMI", - /* 346 */ "ecmd ::= cmdx SEMI", - /* 347 */ "ecmd ::= explain cmdx SEMI", - /* 348 */ "trans_opt ::=", - /* 349 */ "trans_opt ::= TRANSACTION", - /* 350 */ "trans_opt ::= TRANSACTION nm", - /* 351 */ "savepoint_opt ::= SAVEPOINT", - /* 352 */ "savepoint_opt ::=", - /* 353 */ "cmd ::= create_table create_table_args", - /* 354 */ "table_option_set ::= table_option", - /* 355 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 356 */ "columnlist ::= columnname carglist", - /* 357 */ "nm ::= ID|INDEXED", - /* 358 */ "nm ::= STRING", - /* 359 */ "nm ::= JOIN_KW", - /* 360 */ "typetoken ::= typename", - /* 361 */ "typename ::= ID|STRING", - /* 362 */ "signed ::= plus_num", - /* 363 */ "signed ::= minus_num", - /* 364 */ "carglist ::= carglist ccons", - /* 365 */ "carglist ::=", - /* 366 */ "ccons ::= NULL onconf", - /* 367 */ "ccons ::= GENERATED ALWAYS AS generated", - /* 368 */ "ccons ::= AS generated", - /* 369 */ "conslist_opt ::= COMMA conslist", - /* 370 */ "conslist ::= conslist tconscomma tcons", - /* 371 */ "conslist ::= tcons", - /* 372 */ "tconscomma ::=", - /* 373 */ "defer_subclause_opt ::= defer_subclause", - /* 374 */ "resolvetype ::= raisetype", - /* 375 */ "selectnowith ::= oneselect", - /* 376 */ "oneselect ::= values", - /* 377 */ "sclp ::= selcollist COMMA", - /* 378 */ "as ::= ID|STRING", - /* 379 */ "indexed_opt ::= indexed_by", - /* 380 */ "returning ::=", - /* 381 */ "expr ::= term", - /* 382 */ "likeop ::= LIKE_KW|MATCH", - /* 383 */ "exprlist ::= nexprlist", - /* 384 */ "nmnum ::= plus_num", - /* 385 */ "nmnum ::= nm", - /* 386 */ "nmnum ::= ON", - /* 387 */ "nmnum ::= DELETE", - /* 388 */ "nmnum ::= DEFAULT", - /* 389 */ "plus_num ::= INTEGER|FLOAT", - /* 390 */ "foreach_clause ::=", - /* 391 */ "foreach_clause ::= FOR EACH ROW", - /* 392 */ "trnm ::= nm", - /* 393 */ "tridxby ::=", - /* 394 */ "database_kw_opt ::= DATABASE", - /* 395 */ "database_kw_opt ::=", - /* 396 */ "kwcolumn_opt ::=", - /* 397 */ "kwcolumn_opt ::= COLUMNKW", - /* 398 */ "vtabarglist ::= vtabarg", - /* 399 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 400 */ "vtabarg ::= vtabarg vtabargtoken", - /* 401 */ "anylist ::=", - /* 402 */ "anylist ::= anylist LP anylist RP", - /* 403 */ "anylist ::= anylist ANY", - /* 404 */ "with ::=", + /* 178 */ "expr ::= ID|INDEXED|JOIN_KW", + /* 179 */ "expr ::= nm DOT nm", + /* 180 */ "expr ::= nm DOT nm DOT nm", + /* 181 */ "term ::= NULL|FLOAT|BLOB", + /* 182 */ "term ::= STRING", + /* 183 */ "term ::= INTEGER", + /* 184 */ "expr ::= VARIABLE", + /* 185 */ "expr ::= expr COLLATE ID|STRING", + /* 186 */ "expr ::= CAST LP expr AS typetoken RP", + /* 187 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", + /* 188 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", + /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", + /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", + /* 191 */ "term ::= CTIME_KW", + /* 192 */ "expr ::= LP nexprlist COMMA expr RP", + /* 193 */ "expr ::= expr AND expr", + /* 194 */ "expr ::= expr OR expr", + /* 195 */ "expr ::= expr LT|GT|GE|LE expr", + /* 196 */ "expr ::= expr EQ|NE expr", + /* 197 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 198 */ "expr ::= expr PLUS|MINUS expr", + /* 199 */ "expr ::= expr STAR|SLASH|REM expr", + /* 200 */ "expr ::= expr CONCAT expr", + /* 201 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 202 */ "expr ::= expr likeop expr", + /* 203 */ "expr ::= expr likeop expr ESCAPE expr", + /* 204 */ "expr ::= expr ISNULL|NOTNULL", + /* 205 */ "expr ::= expr NOT NULL", + /* 206 */ "expr ::= expr IS expr", + /* 207 */ "expr ::= expr IS NOT expr", + /* 208 */ "expr ::= expr IS NOT DISTINCT FROM expr", + /* 209 */ "expr ::= expr IS DISTINCT FROM expr", + /* 210 */ "expr ::= NOT expr", + /* 211 */ "expr ::= BITNOT expr", + /* 212 */ "expr ::= PLUS|MINUS expr", + /* 213 */ "expr ::= expr PTR expr", + /* 214 */ "between_op ::= BETWEEN", + /* 215 */ "between_op ::= NOT BETWEEN", + /* 216 */ "expr ::= expr between_op expr AND expr", + /* 217 */ "in_op ::= IN", + /* 218 */ "in_op ::= NOT IN", + /* 219 */ "expr ::= expr in_op LP exprlist RP", + /* 220 */ "expr ::= LP select RP", + /* 221 */ "expr ::= expr in_op LP select RP", + /* 222 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 223 */ "expr ::= EXISTS LP select RP", + /* 224 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 225 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 226 */ "case_exprlist ::= WHEN expr THEN expr", + /* 227 */ "case_else ::= ELSE expr", + /* 228 */ "case_else ::=", + /* 229 */ "case_operand ::=", + /* 230 */ "exprlist ::=", + /* 231 */ "nexprlist ::= nexprlist COMMA expr", + /* 232 */ "nexprlist ::= expr", + /* 233 */ "paren_exprlist ::=", + /* 234 */ "paren_exprlist ::= LP exprlist RP", + /* 235 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 236 */ "uniqueflag ::= UNIQUE", + /* 237 */ "uniqueflag ::=", + /* 238 */ "eidlist_opt ::=", + /* 239 */ "eidlist_opt ::= LP eidlist RP", + /* 240 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 241 */ "eidlist ::= nm collate sortorder", + /* 242 */ "collate ::=", + /* 243 */ "collate ::= COLLATE ID|STRING", + /* 244 */ "cmd ::= DROP INDEX ifexists fullname", + /* 245 */ "cmd ::= VACUUM vinto", + /* 246 */ "cmd ::= VACUUM nm vinto", + /* 247 */ "vinto ::= INTO expr", + /* 248 */ "vinto ::=", + /* 249 */ "cmd ::= PRAGMA nm dbnm", + /* 250 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 251 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 252 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 253 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 254 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 255 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 256 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 257 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 258 */ "trigger_time ::= BEFORE|AFTER", + /* 259 */ "trigger_time ::= INSTEAD OF", + /* 260 */ "trigger_time ::=", + /* 261 */ "trigger_event ::= DELETE|INSERT", + /* 262 */ "trigger_event ::= UPDATE", + /* 263 */ "trigger_event ::= UPDATE OF idlist", + /* 264 */ "when_clause ::=", + /* 265 */ "when_clause ::= WHEN expr", + /* 266 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 267 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 268 */ "trnm ::= nm DOT nm", + /* 269 */ "tridxby ::= INDEXED BY nm", + /* 270 */ "tridxby ::= NOT INDEXED", + /* 271 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 272 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 273 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 274 */ "trigger_cmd ::= scanpt select scanpt", + /* 275 */ "expr ::= RAISE LP IGNORE RP", + /* 276 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 277 */ "raisetype ::= ROLLBACK", + /* 278 */ "raisetype ::= ABORT", + /* 279 */ "raisetype ::= FAIL", + /* 280 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 281 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 282 */ "cmd ::= DETACH database_kw_opt expr", + /* 283 */ "key_opt ::=", + /* 284 */ "key_opt ::= KEY expr", + /* 285 */ "cmd ::= REINDEX", + /* 286 */ "cmd ::= REINDEX nm dbnm", + /* 287 */ "cmd ::= ANALYZE", + /* 288 */ "cmd ::= ANALYZE nm dbnm", + /* 289 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 290 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 291 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 292 */ "add_column_fullname ::= fullname", + /* 293 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 294 */ "cmd ::= create_vtab", + /* 295 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 296 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 297 */ "vtabarg ::=", + /* 298 */ "vtabargtoken ::= ANY", + /* 299 */ "vtabargtoken ::= lp anylist RP", + /* 300 */ "lp ::= LP", + /* 301 */ "with ::= WITH wqlist", + /* 302 */ "with ::= WITH RECURSIVE wqlist", + /* 303 */ "wqas ::= AS", + /* 304 */ "wqas ::= AS MATERIALIZED", + /* 305 */ "wqas ::= AS NOT MATERIALIZED", + /* 306 */ "wqitem ::= nm eidlist_opt wqas LP select RP", + /* 307 */ "wqlist ::= wqitem", + /* 308 */ "wqlist ::= wqlist COMMA wqitem", + /* 309 */ "windowdefn_list ::= windowdefn", + /* 310 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 311 */ "windowdefn ::= nm AS LP window RP", + /* 312 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 313 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 314 */ "window ::= ORDER BY sortlist frame_opt", + /* 315 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 316 */ "window ::= frame_opt", + /* 317 */ "window ::= nm frame_opt", + /* 318 */ "frame_opt ::=", + /* 319 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 320 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 321 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 322 */ "frame_bound_s ::= frame_bound", + /* 323 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 324 */ "frame_bound_e ::= frame_bound", + /* 325 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 326 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 327 */ "frame_bound ::= CURRENT ROW", + /* 328 */ "frame_exclude_opt ::=", + /* 329 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 330 */ "frame_exclude ::= NO OTHERS", + /* 331 */ "frame_exclude ::= CURRENT ROW", + /* 332 */ "frame_exclude ::= GROUP|TIES", + /* 333 */ "window_clause ::= WINDOW windowdefn_list", + /* 334 */ "filter_over ::= filter_clause over_clause", + /* 335 */ "filter_over ::= over_clause", + /* 336 */ "filter_over ::= filter_clause", + /* 337 */ "over_clause ::= OVER LP window RP", + /* 338 */ "over_clause ::= OVER nm", + /* 339 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 340 */ "input ::= cmdlist", + /* 341 */ "cmdlist ::= cmdlist ecmd", + /* 342 */ "cmdlist ::= ecmd", + /* 343 */ "ecmd ::= SEMI", + /* 344 */ "ecmd ::= cmdx SEMI", + /* 345 */ "ecmd ::= explain cmdx SEMI", + /* 346 */ "trans_opt ::=", + /* 347 */ "trans_opt ::= TRANSACTION", + /* 348 */ "trans_opt ::= TRANSACTION nm", + /* 349 */ "savepoint_opt ::= SAVEPOINT", + /* 350 */ "savepoint_opt ::=", + /* 351 */ "cmd ::= create_table create_table_args", + /* 352 */ "table_option_set ::= table_option", + /* 353 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 354 */ "columnlist ::= columnname carglist", + /* 355 */ "nm ::= ID|INDEXED|JOIN_KW", + /* 356 */ "nm ::= STRING", + /* 357 */ "typetoken ::= typename", + /* 358 */ "typename ::= ID|STRING", + /* 359 */ "signed ::= plus_num", + /* 360 */ "signed ::= minus_num", + /* 361 */ "carglist ::= carglist ccons", + /* 362 */ "carglist ::=", + /* 363 */ "ccons ::= NULL onconf", + /* 364 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 365 */ "ccons ::= AS generated", + /* 366 */ "conslist_opt ::= COMMA conslist", + /* 367 */ "conslist ::= conslist tconscomma tcons", + /* 368 */ "conslist ::= tcons", + /* 369 */ "tconscomma ::=", + /* 370 */ "defer_subclause_opt ::= defer_subclause", + /* 371 */ "resolvetype ::= raisetype", + /* 372 */ "selectnowith ::= oneselect", + /* 373 */ "oneselect ::= values", + /* 374 */ "sclp ::= selcollist COMMA", + /* 375 */ "as ::= ID|STRING", + /* 376 */ "indexed_opt ::= indexed_by", + /* 377 */ "returning ::=", + /* 378 */ "expr ::= term", + /* 379 */ "likeop ::= LIKE_KW|MATCH", + /* 380 */ "case_operand ::= expr", + /* 381 */ "exprlist ::= nexprlist", + /* 382 */ "nmnum ::= plus_num", + /* 383 */ "nmnum ::= nm", + /* 384 */ "nmnum ::= ON", + /* 385 */ "nmnum ::= DELETE", + /* 386 */ "nmnum ::= DEFAULT", + /* 387 */ "plus_num ::= INTEGER|FLOAT", + /* 388 */ "foreach_clause ::=", + /* 389 */ "foreach_clause ::= FOR EACH ROW", + /* 390 */ "trnm ::= nm", + /* 391 */ "tridxby ::=", + /* 392 */ "database_kw_opt ::= DATABASE", + /* 393 */ "database_kw_opt ::=", + /* 394 */ "kwcolumn_opt ::=", + /* 395 */ "kwcolumn_opt ::= COLUMNKW", + /* 396 */ "vtabarglist ::= vtabarg", + /* 397 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 398 */ "vtabarg ::= vtabarg vtabargtoken", + /* 399 */ "anylist ::=", + /* 400 */ "anylist ::= anylist LP anylist RP", + /* 401 */ "anylist ::= anylist ANY", + /* 402 */ "with ::=", }; #endif /* NDEBUG */ @@ -169682,233 +170623,231 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 263, /* (175) idlist ::= idlist COMMA nm */ 263, /* (176) idlist ::= nm */ 217, /* (177) expr ::= LP expr RP */ - 217, /* (178) expr ::= ID|INDEXED */ - 217, /* (179) expr ::= JOIN_KW */ - 217, /* (180) expr ::= nm DOT nm */ - 217, /* (181) expr ::= nm DOT nm DOT nm */ - 216, /* (182) term ::= NULL|FLOAT|BLOB */ - 216, /* (183) term ::= STRING */ - 216, /* (184) term ::= INTEGER */ - 217, /* (185) expr ::= VARIABLE */ - 217, /* (186) expr ::= expr COLLATE ID|STRING */ - 217, /* (187) expr ::= CAST LP expr AS typetoken RP */ - 217, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */ - 217, /* (189) expr ::= ID|INDEXED LP STAR RP */ - 217, /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ - 217, /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */ - 216, /* (192) term ::= CTIME_KW */ - 217, /* (193) expr ::= LP nexprlist COMMA expr RP */ - 217, /* (194) expr ::= expr AND expr */ - 217, /* (195) expr ::= expr OR expr */ - 217, /* (196) expr ::= expr LT|GT|GE|LE expr */ - 217, /* (197) expr ::= expr EQ|NE expr */ - 217, /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - 217, /* (199) expr ::= expr PLUS|MINUS expr */ - 217, /* (200) expr ::= expr STAR|SLASH|REM expr */ - 217, /* (201) expr ::= expr CONCAT expr */ - 274, /* (202) likeop ::= NOT LIKE_KW|MATCH */ - 217, /* (203) expr ::= expr likeop expr */ - 217, /* (204) expr ::= expr likeop expr ESCAPE expr */ - 217, /* (205) expr ::= expr ISNULL|NOTNULL */ - 217, /* (206) expr ::= expr NOT NULL */ - 217, /* (207) expr ::= expr IS expr */ - 217, /* (208) expr ::= expr IS NOT expr */ - 217, /* (209) expr ::= expr IS NOT DISTINCT FROM expr */ - 217, /* (210) expr ::= expr IS DISTINCT FROM expr */ - 217, /* (211) expr ::= NOT expr */ - 217, /* (212) expr ::= BITNOT expr */ - 217, /* (213) expr ::= PLUS|MINUS expr */ - 217, /* (214) expr ::= expr PTR expr */ - 275, /* (215) between_op ::= BETWEEN */ - 275, /* (216) between_op ::= NOT BETWEEN */ - 217, /* (217) expr ::= expr between_op expr AND expr */ - 276, /* (218) in_op ::= IN */ - 276, /* (219) in_op ::= NOT IN */ - 217, /* (220) expr ::= expr in_op LP exprlist RP */ - 217, /* (221) expr ::= LP select RP */ - 217, /* (222) expr ::= expr in_op LP select RP */ - 217, /* (223) expr ::= expr in_op nm dbnm paren_exprlist */ - 217, /* (224) expr ::= EXISTS LP select RP */ - 217, /* (225) expr ::= CASE case_operand case_exprlist case_else END */ - 279, /* (226) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - 279, /* (227) case_exprlist ::= WHEN expr THEN expr */ - 280, /* (228) case_else ::= ELSE expr */ - 280, /* (229) case_else ::= */ - 278, /* (230) case_operand ::= expr */ - 278, /* (231) case_operand ::= */ - 261, /* (232) exprlist ::= */ - 253, /* (233) nexprlist ::= nexprlist COMMA expr */ - 253, /* (234) nexprlist ::= expr */ - 277, /* (235) paren_exprlist ::= */ - 277, /* (236) paren_exprlist ::= LP exprlist RP */ - 190, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - 281, /* (238) uniqueflag ::= UNIQUE */ - 281, /* (239) uniqueflag ::= */ - 221, /* (240) eidlist_opt ::= */ - 221, /* (241) eidlist_opt ::= LP eidlist RP */ - 232, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - 232, /* (243) eidlist ::= nm collate sortorder */ - 282, /* (244) collate ::= */ - 282, /* (245) collate ::= COLLATE ID|STRING */ - 190, /* (246) cmd ::= DROP INDEX ifexists fullname */ - 190, /* (247) cmd ::= VACUUM vinto */ - 190, /* (248) cmd ::= VACUUM nm vinto */ - 283, /* (249) vinto ::= INTO expr */ - 283, /* (250) vinto ::= */ - 190, /* (251) cmd ::= PRAGMA nm dbnm */ - 190, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - 190, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - 190, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - 190, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - 211, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - 212, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - 190, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - 285, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - 287, /* (260) trigger_time ::= BEFORE|AFTER */ - 287, /* (261) trigger_time ::= INSTEAD OF */ - 287, /* (262) trigger_time ::= */ - 288, /* (263) trigger_event ::= DELETE|INSERT */ - 288, /* (264) trigger_event ::= UPDATE */ - 288, /* (265) trigger_event ::= UPDATE OF idlist */ - 290, /* (266) when_clause ::= */ - 290, /* (267) when_clause ::= WHEN expr */ - 286, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - 286, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - 292, /* (270) trnm ::= nm DOT nm */ - 293, /* (271) tridxby ::= INDEXED BY nm */ - 293, /* (272) tridxby ::= NOT INDEXED */ - 291, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - 291, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - 291, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - 291, /* (276) trigger_cmd ::= scanpt select scanpt */ - 217, /* (277) expr ::= RAISE LP IGNORE RP */ - 217, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - 236, /* (279) raisetype ::= ROLLBACK */ - 236, /* (280) raisetype ::= ABORT */ - 236, /* (281) raisetype ::= FAIL */ - 190, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - 190, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - 190, /* (284) cmd ::= DETACH database_kw_opt expr */ - 295, /* (285) key_opt ::= */ - 295, /* (286) key_opt ::= KEY expr */ - 190, /* (287) cmd ::= REINDEX */ - 190, /* (288) cmd ::= REINDEX nm dbnm */ - 190, /* (289) cmd ::= ANALYZE */ - 190, /* (290) cmd ::= ANALYZE nm dbnm */ - 190, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - 190, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - 190, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - 296, /* (294) add_column_fullname ::= fullname */ - 190, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - 190, /* (296) cmd ::= create_vtab */ - 190, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - 298, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 300, /* (299) vtabarg ::= */ - 301, /* (300) vtabargtoken ::= ANY */ - 301, /* (301) vtabargtoken ::= lp anylist RP */ - 302, /* (302) lp ::= LP */ - 266, /* (303) with ::= WITH wqlist */ - 266, /* (304) with ::= WITH RECURSIVE wqlist */ - 305, /* (305) wqas ::= AS */ - 305, /* (306) wqas ::= AS MATERIALIZED */ - 305, /* (307) wqas ::= AS NOT MATERIALIZED */ - 304, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - 241, /* (309) wqlist ::= wqitem */ - 241, /* (310) wqlist ::= wqlist COMMA wqitem */ - 306, /* (311) windowdefn_list ::= windowdefn */ - 306, /* (312) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - 307, /* (313) windowdefn ::= nm AS LP window RP */ - 308, /* (314) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (315) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - 308, /* (316) window ::= ORDER BY sortlist frame_opt */ - 308, /* (317) window ::= nm ORDER BY sortlist frame_opt */ - 308, /* (318) window ::= frame_opt */ - 308, /* (319) window ::= nm frame_opt */ - 309, /* (320) frame_opt ::= */ - 309, /* (321) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - 309, /* (322) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - 313, /* (323) range_or_rows ::= RANGE|ROWS|GROUPS */ - 315, /* (324) frame_bound_s ::= frame_bound */ - 315, /* (325) frame_bound_s ::= UNBOUNDED PRECEDING */ - 316, /* (326) frame_bound_e ::= frame_bound */ - 316, /* (327) frame_bound_e ::= UNBOUNDED FOLLOWING */ - 314, /* (328) frame_bound ::= expr PRECEDING|FOLLOWING */ - 314, /* (329) frame_bound ::= CURRENT ROW */ - 317, /* (330) frame_exclude_opt ::= */ - 317, /* (331) frame_exclude_opt ::= EXCLUDE frame_exclude */ - 318, /* (332) frame_exclude ::= NO OTHERS */ - 318, /* (333) frame_exclude ::= CURRENT ROW */ - 318, /* (334) frame_exclude ::= GROUP|TIES */ - 251, /* (335) window_clause ::= WINDOW windowdefn_list */ - 273, /* (336) filter_over ::= filter_clause over_clause */ - 273, /* (337) filter_over ::= over_clause */ - 273, /* (338) filter_over ::= filter_clause */ - 312, /* (339) over_clause ::= OVER LP window RP */ - 312, /* (340) over_clause ::= OVER nm */ - 311, /* (341) filter_clause ::= FILTER LP WHERE expr RP */ - 185, /* (342) input ::= cmdlist */ - 186, /* (343) cmdlist ::= cmdlist ecmd */ - 186, /* (344) cmdlist ::= ecmd */ - 187, /* (345) ecmd ::= SEMI */ - 187, /* (346) ecmd ::= cmdx SEMI */ - 187, /* (347) ecmd ::= explain cmdx SEMI */ - 192, /* (348) trans_opt ::= */ - 192, /* (349) trans_opt ::= TRANSACTION */ - 192, /* (350) trans_opt ::= TRANSACTION nm */ - 194, /* (351) savepoint_opt ::= SAVEPOINT */ - 194, /* (352) savepoint_opt ::= */ - 190, /* (353) cmd ::= create_table create_table_args */ - 203, /* (354) table_option_set ::= table_option */ - 201, /* (355) columnlist ::= columnlist COMMA columnname carglist */ - 201, /* (356) columnlist ::= columnname carglist */ - 193, /* (357) nm ::= ID|INDEXED */ - 193, /* (358) nm ::= STRING */ - 193, /* (359) nm ::= JOIN_KW */ - 208, /* (360) typetoken ::= typename */ - 209, /* (361) typename ::= ID|STRING */ - 210, /* (362) signed ::= plus_num */ - 210, /* (363) signed ::= minus_num */ - 207, /* (364) carglist ::= carglist ccons */ - 207, /* (365) carglist ::= */ - 215, /* (366) ccons ::= NULL onconf */ - 215, /* (367) ccons ::= GENERATED ALWAYS AS generated */ - 215, /* (368) ccons ::= AS generated */ - 202, /* (369) conslist_opt ::= COMMA conslist */ - 228, /* (370) conslist ::= conslist tconscomma tcons */ - 228, /* (371) conslist ::= tcons */ - 229, /* (372) tconscomma ::= */ - 233, /* (373) defer_subclause_opt ::= defer_subclause */ - 235, /* (374) resolvetype ::= raisetype */ - 239, /* (375) selectnowith ::= oneselect */ - 240, /* (376) oneselect ::= values */ - 254, /* (377) sclp ::= selcollist COMMA */ - 255, /* (378) as ::= ID|STRING */ - 264, /* (379) indexed_opt ::= indexed_by */ - 272, /* (380) returning ::= */ - 217, /* (381) expr ::= term */ - 274, /* (382) likeop ::= LIKE_KW|MATCH */ - 261, /* (383) exprlist ::= nexprlist */ - 284, /* (384) nmnum ::= plus_num */ - 284, /* (385) nmnum ::= nm */ - 284, /* (386) nmnum ::= ON */ - 284, /* (387) nmnum ::= DELETE */ - 284, /* (388) nmnum ::= DEFAULT */ - 211, /* (389) plus_num ::= INTEGER|FLOAT */ - 289, /* (390) foreach_clause ::= */ - 289, /* (391) foreach_clause ::= FOR EACH ROW */ - 292, /* (392) trnm ::= nm */ - 293, /* (393) tridxby ::= */ - 294, /* (394) database_kw_opt ::= DATABASE */ - 294, /* (395) database_kw_opt ::= */ - 297, /* (396) kwcolumn_opt ::= */ - 297, /* (397) kwcolumn_opt ::= COLUMNKW */ - 299, /* (398) vtabarglist ::= vtabarg */ - 299, /* (399) vtabarglist ::= vtabarglist COMMA vtabarg */ - 300, /* (400) vtabarg ::= vtabarg vtabargtoken */ - 303, /* (401) anylist ::= */ - 303, /* (402) anylist ::= anylist LP anylist RP */ - 303, /* (403) anylist ::= anylist ANY */ - 266, /* (404) with ::= */ + 217, /* (178) expr ::= ID|INDEXED|JOIN_KW */ + 217, /* (179) expr ::= nm DOT nm */ + 217, /* (180) expr ::= nm DOT nm DOT nm */ + 216, /* (181) term ::= NULL|FLOAT|BLOB */ + 216, /* (182) term ::= STRING */ + 216, /* (183) term ::= INTEGER */ + 217, /* (184) expr ::= VARIABLE */ + 217, /* (185) expr ::= expr COLLATE ID|STRING */ + 217, /* (186) expr ::= CAST LP expr AS typetoken RP */ + 217, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + 217, /* (188) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + 217, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + 217, /* (190) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + 216, /* (191) term ::= CTIME_KW */ + 217, /* (192) expr ::= LP nexprlist COMMA expr RP */ + 217, /* (193) expr ::= expr AND expr */ + 217, /* (194) expr ::= expr OR expr */ + 217, /* (195) expr ::= expr LT|GT|GE|LE expr */ + 217, /* (196) expr ::= expr EQ|NE expr */ + 217, /* (197) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 217, /* (198) expr ::= expr PLUS|MINUS expr */ + 217, /* (199) expr ::= expr STAR|SLASH|REM expr */ + 217, /* (200) expr ::= expr CONCAT expr */ + 274, /* (201) likeop ::= NOT LIKE_KW|MATCH */ + 217, /* (202) expr ::= expr likeop expr */ + 217, /* (203) expr ::= expr likeop expr ESCAPE expr */ + 217, /* (204) expr ::= expr ISNULL|NOTNULL */ + 217, /* (205) expr ::= expr NOT NULL */ + 217, /* (206) expr ::= expr IS expr */ + 217, /* (207) expr ::= expr IS NOT expr */ + 217, /* (208) expr ::= expr IS NOT DISTINCT FROM expr */ + 217, /* (209) expr ::= expr IS DISTINCT FROM expr */ + 217, /* (210) expr ::= NOT expr */ + 217, /* (211) expr ::= BITNOT expr */ + 217, /* (212) expr ::= PLUS|MINUS expr */ + 217, /* (213) expr ::= expr PTR expr */ + 275, /* (214) between_op ::= BETWEEN */ + 275, /* (215) between_op ::= NOT BETWEEN */ + 217, /* (216) expr ::= expr between_op expr AND expr */ + 276, /* (217) in_op ::= IN */ + 276, /* (218) in_op ::= NOT IN */ + 217, /* (219) expr ::= expr in_op LP exprlist RP */ + 217, /* (220) expr ::= LP select RP */ + 217, /* (221) expr ::= expr in_op LP select RP */ + 217, /* (222) expr ::= expr in_op nm dbnm paren_exprlist */ + 217, /* (223) expr ::= EXISTS LP select RP */ + 217, /* (224) expr ::= CASE case_operand case_exprlist case_else END */ + 279, /* (225) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 279, /* (226) case_exprlist ::= WHEN expr THEN expr */ + 280, /* (227) case_else ::= ELSE expr */ + 280, /* (228) case_else ::= */ + 278, /* (229) case_operand ::= */ + 261, /* (230) exprlist ::= */ + 253, /* (231) nexprlist ::= nexprlist COMMA expr */ + 253, /* (232) nexprlist ::= expr */ + 277, /* (233) paren_exprlist ::= */ + 277, /* (234) paren_exprlist ::= LP exprlist RP */ + 190, /* (235) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 281, /* (236) uniqueflag ::= UNIQUE */ + 281, /* (237) uniqueflag ::= */ + 221, /* (238) eidlist_opt ::= */ + 221, /* (239) eidlist_opt ::= LP eidlist RP */ + 232, /* (240) eidlist ::= eidlist COMMA nm collate sortorder */ + 232, /* (241) eidlist ::= nm collate sortorder */ + 282, /* (242) collate ::= */ + 282, /* (243) collate ::= COLLATE ID|STRING */ + 190, /* (244) cmd ::= DROP INDEX ifexists fullname */ + 190, /* (245) cmd ::= VACUUM vinto */ + 190, /* (246) cmd ::= VACUUM nm vinto */ + 283, /* (247) vinto ::= INTO expr */ + 283, /* (248) vinto ::= */ + 190, /* (249) cmd ::= PRAGMA nm dbnm */ + 190, /* (250) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 190, /* (251) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 190, /* (252) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 190, /* (253) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 211, /* (254) plus_num ::= PLUS INTEGER|FLOAT */ + 212, /* (255) minus_num ::= MINUS INTEGER|FLOAT */ + 190, /* (256) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 285, /* (257) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 287, /* (258) trigger_time ::= BEFORE|AFTER */ + 287, /* (259) trigger_time ::= INSTEAD OF */ + 287, /* (260) trigger_time ::= */ + 288, /* (261) trigger_event ::= DELETE|INSERT */ + 288, /* (262) trigger_event ::= UPDATE */ + 288, /* (263) trigger_event ::= UPDATE OF idlist */ + 290, /* (264) when_clause ::= */ + 290, /* (265) when_clause ::= WHEN expr */ + 286, /* (266) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 286, /* (267) trigger_cmd_list ::= trigger_cmd SEMI */ + 292, /* (268) trnm ::= nm DOT nm */ + 293, /* (269) tridxby ::= INDEXED BY nm */ + 293, /* (270) tridxby ::= NOT INDEXED */ + 291, /* (271) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 291, /* (272) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 291, /* (273) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 291, /* (274) trigger_cmd ::= scanpt select scanpt */ + 217, /* (275) expr ::= RAISE LP IGNORE RP */ + 217, /* (276) expr ::= RAISE LP raisetype COMMA nm RP */ + 236, /* (277) raisetype ::= ROLLBACK */ + 236, /* (278) raisetype ::= ABORT */ + 236, /* (279) raisetype ::= FAIL */ + 190, /* (280) cmd ::= DROP TRIGGER ifexists fullname */ + 190, /* (281) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 190, /* (282) cmd ::= DETACH database_kw_opt expr */ + 295, /* (283) key_opt ::= */ + 295, /* (284) key_opt ::= KEY expr */ + 190, /* (285) cmd ::= REINDEX */ + 190, /* (286) cmd ::= REINDEX nm dbnm */ + 190, /* (287) cmd ::= ANALYZE */ + 190, /* (288) cmd ::= ANALYZE nm dbnm */ + 190, /* (289) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 190, /* (290) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 190, /* (291) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 296, /* (292) add_column_fullname ::= fullname */ + 190, /* (293) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 190, /* (294) cmd ::= create_vtab */ + 190, /* (295) cmd ::= create_vtab LP vtabarglist RP */ + 298, /* (296) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 300, /* (297) vtabarg ::= */ + 301, /* (298) vtabargtoken ::= ANY */ + 301, /* (299) vtabargtoken ::= lp anylist RP */ + 302, /* (300) lp ::= LP */ + 266, /* (301) with ::= WITH wqlist */ + 266, /* (302) with ::= WITH RECURSIVE wqlist */ + 305, /* (303) wqas ::= AS */ + 305, /* (304) wqas ::= AS MATERIALIZED */ + 305, /* (305) wqas ::= AS NOT MATERIALIZED */ + 304, /* (306) wqitem ::= nm eidlist_opt wqas LP select RP */ + 241, /* (307) wqlist ::= wqitem */ + 241, /* (308) wqlist ::= wqlist COMMA wqitem */ + 306, /* (309) windowdefn_list ::= windowdefn */ + 306, /* (310) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 307, /* (311) windowdefn ::= nm AS LP window RP */ + 308, /* (312) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 308, /* (313) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 308, /* (314) window ::= ORDER BY sortlist frame_opt */ + 308, /* (315) window ::= nm ORDER BY sortlist frame_opt */ + 308, /* (316) window ::= frame_opt */ + 308, /* (317) window ::= nm frame_opt */ + 309, /* (318) frame_opt ::= */ + 309, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 309, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 313, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ + 315, /* (322) frame_bound_s ::= frame_bound */ + 315, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ + 316, /* (324) frame_bound_e ::= frame_bound */ + 316, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 314, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ + 314, /* (327) frame_bound ::= CURRENT ROW */ + 317, /* (328) frame_exclude_opt ::= */ + 317, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 318, /* (330) frame_exclude ::= NO OTHERS */ + 318, /* (331) frame_exclude ::= CURRENT ROW */ + 318, /* (332) frame_exclude ::= GROUP|TIES */ + 251, /* (333) window_clause ::= WINDOW windowdefn_list */ + 273, /* (334) filter_over ::= filter_clause over_clause */ + 273, /* (335) filter_over ::= over_clause */ + 273, /* (336) filter_over ::= filter_clause */ + 312, /* (337) over_clause ::= OVER LP window RP */ + 312, /* (338) over_clause ::= OVER nm */ + 311, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ + 185, /* (340) input ::= cmdlist */ + 186, /* (341) cmdlist ::= cmdlist ecmd */ + 186, /* (342) cmdlist ::= ecmd */ + 187, /* (343) ecmd ::= SEMI */ + 187, /* (344) ecmd ::= cmdx SEMI */ + 187, /* (345) ecmd ::= explain cmdx SEMI */ + 192, /* (346) trans_opt ::= */ + 192, /* (347) trans_opt ::= TRANSACTION */ + 192, /* (348) trans_opt ::= TRANSACTION nm */ + 194, /* (349) savepoint_opt ::= SAVEPOINT */ + 194, /* (350) savepoint_opt ::= */ + 190, /* (351) cmd ::= create_table create_table_args */ + 203, /* (352) table_option_set ::= table_option */ + 201, /* (353) columnlist ::= columnlist COMMA columnname carglist */ + 201, /* (354) columnlist ::= columnname carglist */ + 193, /* (355) nm ::= ID|INDEXED|JOIN_KW */ + 193, /* (356) nm ::= STRING */ + 208, /* (357) typetoken ::= typename */ + 209, /* (358) typename ::= ID|STRING */ + 210, /* (359) signed ::= plus_num */ + 210, /* (360) signed ::= minus_num */ + 207, /* (361) carglist ::= carglist ccons */ + 207, /* (362) carglist ::= */ + 215, /* (363) ccons ::= NULL onconf */ + 215, /* (364) ccons ::= GENERATED ALWAYS AS generated */ + 215, /* (365) ccons ::= AS generated */ + 202, /* (366) conslist_opt ::= COMMA conslist */ + 228, /* (367) conslist ::= conslist tconscomma tcons */ + 228, /* (368) conslist ::= tcons */ + 229, /* (369) tconscomma ::= */ + 233, /* (370) defer_subclause_opt ::= defer_subclause */ + 235, /* (371) resolvetype ::= raisetype */ + 239, /* (372) selectnowith ::= oneselect */ + 240, /* (373) oneselect ::= values */ + 254, /* (374) sclp ::= selcollist COMMA */ + 255, /* (375) as ::= ID|STRING */ + 264, /* (376) indexed_opt ::= indexed_by */ + 272, /* (377) returning ::= */ + 217, /* (378) expr ::= term */ + 274, /* (379) likeop ::= LIKE_KW|MATCH */ + 278, /* (380) case_operand ::= expr */ + 261, /* (381) exprlist ::= nexprlist */ + 284, /* (382) nmnum ::= plus_num */ + 284, /* (383) nmnum ::= nm */ + 284, /* (384) nmnum ::= ON */ + 284, /* (385) nmnum ::= DELETE */ + 284, /* (386) nmnum ::= DEFAULT */ + 211, /* (387) plus_num ::= INTEGER|FLOAT */ + 289, /* (388) foreach_clause ::= */ + 289, /* (389) foreach_clause ::= FOR EACH ROW */ + 292, /* (390) trnm ::= nm */ + 293, /* (391) tridxby ::= */ + 294, /* (392) database_kw_opt ::= DATABASE */ + 294, /* (393) database_kw_opt ::= */ + 297, /* (394) kwcolumn_opt ::= */ + 297, /* (395) kwcolumn_opt ::= COLUMNKW */ + 299, /* (396) vtabarglist ::= vtabarg */ + 299, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ + 300, /* (398) vtabarg ::= vtabarg vtabargtoken */ + 303, /* (399) anylist ::= */ + 303, /* (400) anylist ::= anylist LP anylist RP */ + 303, /* (401) anylist ::= anylist ANY */ + 266, /* (402) with ::= */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -170092,233 +171031,231 @@ static const signed char yyRuleInfoNRhs[] = { -3, /* (175) idlist ::= idlist COMMA nm */ -1, /* (176) idlist ::= nm */ -3, /* (177) expr ::= LP expr RP */ - -1, /* (178) expr ::= ID|INDEXED */ - -1, /* (179) expr ::= JOIN_KW */ - -3, /* (180) expr ::= nm DOT nm */ - -5, /* (181) expr ::= nm DOT nm DOT nm */ - -1, /* (182) term ::= NULL|FLOAT|BLOB */ - -1, /* (183) term ::= STRING */ - -1, /* (184) term ::= INTEGER */ - -1, /* (185) expr ::= VARIABLE */ - -3, /* (186) expr ::= expr COLLATE ID|STRING */ - -6, /* (187) expr ::= CAST LP expr AS typetoken RP */ - -5, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */ - -4, /* (189) expr ::= ID|INDEXED LP STAR RP */ - -6, /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ - -5, /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */ - -1, /* (192) term ::= CTIME_KW */ - -5, /* (193) expr ::= LP nexprlist COMMA expr RP */ - -3, /* (194) expr ::= expr AND expr */ - -3, /* (195) expr ::= expr OR expr */ - -3, /* (196) expr ::= expr LT|GT|GE|LE expr */ - -3, /* (197) expr ::= expr EQ|NE expr */ - -3, /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - -3, /* (199) expr ::= expr PLUS|MINUS expr */ - -3, /* (200) expr ::= expr STAR|SLASH|REM expr */ - -3, /* (201) expr ::= expr CONCAT expr */ - -2, /* (202) likeop ::= NOT LIKE_KW|MATCH */ - -3, /* (203) expr ::= expr likeop expr */ - -5, /* (204) expr ::= expr likeop expr ESCAPE expr */ - -2, /* (205) expr ::= expr ISNULL|NOTNULL */ - -3, /* (206) expr ::= expr NOT NULL */ - -3, /* (207) expr ::= expr IS expr */ - -4, /* (208) expr ::= expr IS NOT expr */ - -6, /* (209) expr ::= expr IS NOT DISTINCT FROM expr */ - -5, /* (210) expr ::= expr IS DISTINCT FROM expr */ - -2, /* (211) expr ::= NOT expr */ - -2, /* (212) expr ::= BITNOT expr */ - -2, /* (213) expr ::= PLUS|MINUS expr */ - -3, /* (214) expr ::= expr PTR expr */ - -1, /* (215) between_op ::= BETWEEN */ - -2, /* (216) between_op ::= NOT BETWEEN */ - -5, /* (217) expr ::= expr between_op expr AND expr */ - -1, /* (218) in_op ::= IN */ - -2, /* (219) in_op ::= NOT IN */ - -5, /* (220) expr ::= expr in_op LP exprlist RP */ - -3, /* (221) expr ::= LP select RP */ - -5, /* (222) expr ::= expr in_op LP select RP */ - -5, /* (223) expr ::= expr in_op nm dbnm paren_exprlist */ - -4, /* (224) expr ::= EXISTS LP select RP */ - -5, /* (225) expr ::= CASE case_operand case_exprlist case_else END */ - -5, /* (226) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - -4, /* (227) case_exprlist ::= WHEN expr THEN expr */ - -2, /* (228) case_else ::= ELSE expr */ - 0, /* (229) case_else ::= */ - -1, /* (230) case_operand ::= expr */ - 0, /* (231) case_operand ::= */ - 0, /* (232) exprlist ::= */ - -3, /* (233) nexprlist ::= nexprlist COMMA expr */ - -1, /* (234) nexprlist ::= expr */ - 0, /* (235) paren_exprlist ::= */ - -3, /* (236) paren_exprlist ::= LP exprlist RP */ - -12, /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - -1, /* (238) uniqueflag ::= UNIQUE */ - 0, /* (239) uniqueflag ::= */ - 0, /* (240) eidlist_opt ::= */ - -3, /* (241) eidlist_opt ::= LP eidlist RP */ - -5, /* (242) eidlist ::= eidlist COMMA nm collate sortorder */ - -3, /* (243) eidlist ::= nm collate sortorder */ - 0, /* (244) collate ::= */ - -2, /* (245) collate ::= COLLATE ID|STRING */ - -4, /* (246) cmd ::= DROP INDEX ifexists fullname */ - -2, /* (247) cmd ::= VACUUM vinto */ - -3, /* (248) cmd ::= VACUUM nm vinto */ - -2, /* (249) vinto ::= INTO expr */ - 0, /* (250) vinto ::= */ - -3, /* (251) cmd ::= PRAGMA nm dbnm */ - -5, /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */ - -6, /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - -5, /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */ - -6, /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - -2, /* (256) plus_num ::= PLUS INTEGER|FLOAT */ - -2, /* (257) minus_num ::= MINUS INTEGER|FLOAT */ - -5, /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - -11, /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - -1, /* (260) trigger_time ::= BEFORE|AFTER */ - -2, /* (261) trigger_time ::= INSTEAD OF */ - 0, /* (262) trigger_time ::= */ - -1, /* (263) trigger_event ::= DELETE|INSERT */ - -1, /* (264) trigger_event ::= UPDATE */ - -3, /* (265) trigger_event ::= UPDATE OF idlist */ - 0, /* (266) when_clause ::= */ - -2, /* (267) when_clause ::= WHEN expr */ - -3, /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - -2, /* (269) trigger_cmd_list ::= trigger_cmd SEMI */ - -3, /* (270) trnm ::= nm DOT nm */ - -3, /* (271) tridxby ::= INDEXED BY nm */ - -2, /* (272) tridxby ::= NOT INDEXED */ - -9, /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ - -8, /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - -6, /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - -3, /* (276) trigger_cmd ::= scanpt select scanpt */ - -4, /* (277) expr ::= RAISE LP IGNORE RP */ - -6, /* (278) expr ::= RAISE LP raisetype COMMA nm RP */ - -1, /* (279) raisetype ::= ROLLBACK */ - -1, /* (280) raisetype ::= ABORT */ - -1, /* (281) raisetype ::= FAIL */ - -4, /* (282) cmd ::= DROP TRIGGER ifexists fullname */ - -6, /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - -3, /* (284) cmd ::= DETACH database_kw_opt expr */ - 0, /* (285) key_opt ::= */ - -2, /* (286) key_opt ::= KEY expr */ - -1, /* (287) cmd ::= REINDEX */ - -3, /* (288) cmd ::= REINDEX nm dbnm */ - -1, /* (289) cmd ::= ANALYZE */ - -3, /* (290) cmd ::= ANALYZE nm dbnm */ - -6, /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */ - -7, /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - -6, /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ - -1, /* (294) add_column_fullname ::= fullname */ - -8, /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ - -1, /* (296) cmd ::= create_vtab */ - -4, /* (297) cmd ::= create_vtab LP vtabarglist RP */ - -8, /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - 0, /* (299) vtabarg ::= */ - -1, /* (300) vtabargtoken ::= ANY */ - -3, /* (301) vtabargtoken ::= lp anylist RP */ - -1, /* (302) lp ::= LP */ - -2, /* (303) with ::= WITH wqlist */ - -3, /* (304) with ::= WITH RECURSIVE wqlist */ - -1, /* (305) wqas ::= AS */ - -2, /* (306) wqas ::= AS MATERIALIZED */ - -3, /* (307) wqas ::= AS NOT MATERIALIZED */ - -6, /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */ - -1, /* (309) wqlist ::= wqitem */ - -3, /* (310) wqlist ::= wqlist COMMA wqitem */ - -1, /* (311) windowdefn_list ::= windowdefn */ - -3, /* (312) windowdefn_list ::= windowdefn_list COMMA windowdefn */ - -5, /* (313) windowdefn ::= nm AS LP window RP */ - -5, /* (314) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ - -6, /* (315) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ - -4, /* (316) window ::= ORDER BY sortlist frame_opt */ - -5, /* (317) window ::= nm ORDER BY sortlist frame_opt */ - -1, /* (318) window ::= frame_opt */ - -2, /* (319) window ::= nm frame_opt */ - 0, /* (320) frame_opt ::= */ - -3, /* (321) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ - -6, /* (322) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ - -1, /* (323) range_or_rows ::= RANGE|ROWS|GROUPS */ - -1, /* (324) frame_bound_s ::= frame_bound */ - -2, /* (325) frame_bound_s ::= UNBOUNDED PRECEDING */ - -1, /* (326) frame_bound_e ::= frame_bound */ - -2, /* (327) frame_bound_e ::= UNBOUNDED FOLLOWING */ - -2, /* (328) frame_bound ::= expr PRECEDING|FOLLOWING */ - -2, /* (329) frame_bound ::= CURRENT ROW */ - 0, /* (330) frame_exclude_opt ::= */ - -2, /* (331) frame_exclude_opt ::= EXCLUDE frame_exclude */ - -2, /* (332) frame_exclude ::= NO OTHERS */ - -2, /* (333) frame_exclude ::= CURRENT ROW */ - -1, /* (334) frame_exclude ::= GROUP|TIES */ - -2, /* (335) window_clause ::= WINDOW windowdefn_list */ - -2, /* (336) filter_over ::= filter_clause over_clause */ - -1, /* (337) filter_over ::= over_clause */ - -1, /* (338) filter_over ::= filter_clause */ - -4, /* (339) over_clause ::= OVER LP window RP */ - -2, /* (340) over_clause ::= OVER nm */ - -5, /* (341) filter_clause ::= FILTER LP WHERE expr RP */ - -1, /* (342) input ::= cmdlist */ - -2, /* (343) cmdlist ::= cmdlist ecmd */ - -1, /* (344) cmdlist ::= ecmd */ - -1, /* (345) ecmd ::= SEMI */ - -2, /* (346) ecmd ::= cmdx SEMI */ - -3, /* (347) ecmd ::= explain cmdx SEMI */ - 0, /* (348) trans_opt ::= */ - -1, /* (349) trans_opt ::= TRANSACTION */ - -2, /* (350) trans_opt ::= TRANSACTION nm */ - -1, /* (351) savepoint_opt ::= SAVEPOINT */ - 0, /* (352) savepoint_opt ::= */ - -2, /* (353) cmd ::= create_table create_table_args */ - -1, /* (354) table_option_set ::= table_option */ - -4, /* (355) columnlist ::= columnlist COMMA columnname carglist */ - -2, /* (356) columnlist ::= columnname carglist */ - -1, /* (357) nm ::= ID|INDEXED */ - -1, /* (358) nm ::= STRING */ - -1, /* (359) nm ::= JOIN_KW */ - -1, /* (360) typetoken ::= typename */ - -1, /* (361) typename ::= ID|STRING */ - -1, /* (362) signed ::= plus_num */ - -1, /* (363) signed ::= minus_num */ - -2, /* (364) carglist ::= carglist ccons */ - 0, /* (365) carglist ::= */ - -2, /* (366) ccons ::= NULL onconf */ - -4, /* (367) ccons ::= GENERATED ALWAYS AS generated */ - -2, /* (368) ccons ::= AS generated */ - -2, /* (369) conslist_opt ::= COMMA conslist */ - -3, /* (370) conslist ::= conslist tconscomma tcons */ - -1, /* (371) conslist ::= tcons */ - 0, /* (372) tconscomma ::= */ - -1, /* (373) defer_subclause_opt ::= defer_subclause */ - -1, /* (374) resolvetype ::= raisetype */ - -1, /* (375) selectnowith ::= oneselect */ - -1, /* (376) oneselect ::= values */ - -2, /* (377) sclp ::= selcollist COMMA */ - -1, /* (378) as ::= ID|STRING */ - -1, /* (379) indexed_opt ::= indexed_by */ - 0, /* (380) returning ::= */ - -1, /* (381) expr ::= term */ - -1, /* (382) likeop ::= LIKE_KW|MATCH */ - -1, /* (383) exprlist ::= nexprlist */ - -1, /* (384) nmnum ::= plus_num */ - -1, /* (385) nmnum ::= nm */ - -1, /* (386) nmnum ::= ON */ - -1, /* (387) nmnum ::= DELETE */ - -1, /* (388) nmnum ::= DEFAULT */ - -1, /* (389) plus_num ::= INTEGER|FLOAT */ - 0, /* (390) foreach_clause ::= */ - -3, /* (391) foreach_clause ::= FOR EACH ROW */ - -1, /* (392) trnm ::= nm */ - 0, /* (393) tridxby ::= */ - -1, /* (394) database_kw_opt ::= DATABASE */ - 0, /* (395) database_kw_opt ::= */ - 0, /* (396) kwcolumn_opt ::= */ - -1, /* (397) kwcolumn_opt ::= COLUMNKW */ - -1, /* (398) vtabarglist ::= vtabarg */ - -3, /* (399) vtabarglist ::= vtabarglist COMMA vtabarg */ - -2, /* (400) vtabarg ::= vtabarg vtabargtoken */ - 0, /* (401) anylist ::= */ - -4, /* (402) anylist ::= anylist LP anylist RP */ - -2, /* (403) anylist ::= anylist ANY */ - 0, /* (404) with ::= */ + -1, /* (178) expr ::= ID|INDEXED|JOIN_KW */ + -3, /* (179) expr ::= nm DOT nm */ + -5, /* (180) expr ::= nm DOT nm DOT nm */ + -1, /* (181) term ::= NULL|FLOAT|BLOB */ + -1, /* (182) term ::= STRING */ + -1, /* (183) term ::= INTEGER */ + -1, /* (184) expr ::= VARIABLE */ + -3, /* (185) expr ::= expr COLLATE ID|STRING */ + -6, /* (186) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + -4, /* (188) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + -6, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + -5, /* (190) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + -1, /* (191) term ::= CTIME_KW */ + -5, /* (192) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (193) expr ::= expr AND expr */ + -3, /* (194) expr ::= expr OR expr */ + -3, /* (195) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (196) expr ::= expr EQ|NE expr */ + -3, /* (197) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (198) expr ::= expr PLUS|MINUS expr */ + -3, /* (199) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (200) expr ::= expr CONCAT expr */ + -2, /* (201) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (202) expr ::= expr likeop expr */ + -5, /* (203) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (204) expr ::= expr ISNULL|NOTNULL */ + -3, /* (205) expr ::= expr NOT NULL */ + -3, /* (206) expr ::= expr IS expr */ + -4, /* (207) expr ::= expr IS NOT expr */ + -6, /* (208) expr ::= expr IS NOT DISTINCT FROM expr */ + -5, /* (209) expr ::= expr IS DISTINCT FROM expr */ + -2, /* (210) expr ::= NOT expr */ + -2, /* (211) expr ::= BITNOT expr */ + -2, /* (212) expr ::= PLUS|MINUS expr */ + -3, /* (213) expr ::= expr PTR expr */ + -1, /* (214) between_op ::= BETWEEN */ + -2, /* (215) between_op ::= NOT BETWEEN */ + -5, /* (216) expr ::= expr between_op expr AND expr */ + -1, /* (217) in_op ::= IN */ + -2, /* (218) in_op ::= NOT IN */ + -5, /* (219) expr ::= expr in_op LP exprlist RP */ + -3, /* (220) expr ::= LP select RP */ + -5, /* (221) expr ::= expr in_op LP select RP */ + -5, /* (222) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (223) expr ::= EXISTS LP select RP */ + -5, /* (224) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (225) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (226) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (227) case_else ::= ELSE expr */ + 0, /* (228) case_else ::= */ + 0, /* (229) case_operand ::= */ + 0, /* (230) exprlist ::= */ + -3, /* (231) nexprlist ::= nexprlist COMMA expr */ + -1, /* (232) nexprlist ::= expr */ + 0, /* (233) paren_exprlist ::= */ + -3, /* (234) paren_exprlist ::= LP exprlist RP */ + -12, /* (235) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (236) uniqueflag ::= UNIQUE */ + 0, /* (237) uniqueflag ::= */ + 0, /* (238) eidlist_opt ::= */ + -3, /* (239) eidlist_opt ::= LP eidlist RP */ + -5, /* (240) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (241) eidlist ::= nm collate sortorder */ + 0, /* (242) collate ::= */ + -2, /* (243) collate ::= COLLATE ID|STRING */ + -4, /* (244) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (245) cmd ::= VACUUM vinto */ + -3, /* (246) cmd ::= VACUUM nm vinto */ + -2, /* (247) vinto ::= INTO expr */ + 0, /* (248) vinto ::= */ + -3, /* (249) cmd ::= PRAGMA nm dbnm */ + -5, /* (250) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (251) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (252) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (253) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (254) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (255) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (256) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (257) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (258) trigger_time ::= BEFORE|AFTER */ + -2, /* (259) trigger_time ::= INSTEAD OF */ + 0, /* (260) trigger_time ::= */ + -1, /* (261) trigger_event ::= DELETE|INSERT */ + -1, /* (262) trigger_event ::= UPDATE */ + -3, /* (263) trigger_event ::= UPDATE OF idlist */ + 0, /* (264) when_clause ::= */ + -2, /* (265) when_clause ::= WHEN expr */ + -3, /* (266) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (267) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (268) trnm ::= nm DOT nm */ + -3, /* (269) tridxby ::= INDEXED BY nm */ + -2, /* (270) tridxby ::= NOT INDEXED */ + -9, /* (271) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (272) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (273) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (274) trigger_cmd ::= scanpt select scanpt */ + -4, /* (275) expr ::= RAISE LP IGNORE RP */ + -6, /* (276) expr ::= RAISE LP raisetype COMMA nm RP */ + -1, /* (277) raisetype ::= ROLLBACK */ + -1, /* (278) raisetype ::= ABORT */ + -1, /* (279) raisetype ::= FAIL */ + -4, /* (280) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (281) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (282) cmd ::= DETACH database_kw_opt expr */ + 0, /* (283) key_opt ::= */ + -2, /* (284) key_opt ::= KEY expr */ + -1, /* (285) cmd ::= REINDEX */ + -3, /* (286) cmd ::= REINDEX nm dbnm */ + -1, /* (287) cmd ::= ANALYZE */ + -3, /* (288) cmd ::= ANALYZE nm dbnm */ + -6, /* (289) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (290) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (291) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (292) add_column_fullname ::= fullname */ + -8, /* (293) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (294) cmd ::= create_vtab */ + -4, /* (295) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (296) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (297) vtabarg ::= */ + -1, /* (298) vtabargtoken ::= ANY */ + -3, /* (299) vtabargtoken ::= lp anylist RP */ + -1, /* (300) lp ::= LP */ + -2, /* (301) with ::= WITH wqlist */ + -3, /* (302) with ::= WITH RECURSIVE wqlist */ + -1, /* (303) wqas ::= AS */ + -2, /* (304) wqas ::= AS MATERIALIZED */ + -3, /* (305) wqas ::= AS NOT MATERIALIZED */ + -6, /* (306) wqitem ::= nm eidlist_opt wqas LP select RP */ + -1, /* (307) wqlist ::= wqitem */ + -3, /* (308) wqlist ::= wqlist COMMA wqitem */ + -1, /* (309) windowdefn_list ::= windowdefn */ + -3, /* (310) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (311) windowdefn ::= nm AS LP window RP */ + -5, /* (312) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (313) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (314) window ::= ORDER BY sortlist frame_opt */ + -5, /* (315) window ::= nm ORDER BY sortlist frame_opt */ + -1, /* (316) window ::= frame_opt */ + -2, /* (317) window ::= nm frame_opt */ + 0, /* (318) frame_opt ::= */ + -3, /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (322) frame_bound_s ::= frame_bound */ + -2, /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (324) frame_bound_e ::= frame_bound */ + -2, /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (327) frame_bound ::= CURRENT ROW */ + 0, /* (328) frame_exclude_opt ::= */ + -2, /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (330) frame_exclude ::= NO OTHERS */ + -2, /* (331) frame_exclude ::= CURRENT ROW */ + -1, /* (332) frame_exclude ::= GROUP|TIES */ + -2, /* (333) window_clause ::= WINDOW windowdefn_list */ + -2, /* (334) filter_over ::= filter_clause over_clause */ + -1, /* (335) filter_over ::= over_clause */ + -1, /* (336) filter_over ::= filter_clause */ + -4, /* (337) over_clause ::= OVER LP window RP */ + -2, /* (338) over_clause ::= OVER nm */ + -5, /* (339) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (340) input ::= cmdlist */ + -2, /* (341) cmdlist ::= cmdlist ecmd */ + -1, /* (342) cmdlist ::= ecmd */ + -1, /* (343) ecmd ::= SEMI */ + -2, /* (344) ecmd ::= cmdx SEMI */ + -3, /* (345) ecmd ::= explain cmdx SEMI */ + 0, /* (346) trans_opt ::= */ + -1, /* (347) trans_opt ::= TRANSACTION */ + -2, /* (348) trans_opt ::= TRANSACTION nm */ + -1, /* (349) savepoint_opt ::= SAVEPOINT */ + 0, /* (350) savepoint_opt ::= */ + -2, /* (351) cmd ::= create_table create_table_args */ + -1, /* (352) table_option_set ::= table_option */ + -4, /* (353) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (354) columnlist ::= columnname carglist */ + -1, /* (355) nm ::= ID|INDEXED|JOIN_KW */ + -1, /* (356) nm ::= STRING */ + -1, /* (357) typetoken ::= typename */ + -1, /* (358) typename ::= ID|STRING */ + -1, /* (359) signed ::= plus_num */ + -1, /* (360) signed ::= minus_num */ + -2, /* (361) carglist ::= carglist ccons */ + 0, /* (362) carglist ::= */ + -2, /* (363) ccons ::= NULL onconf */ + -4, /* (364) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (365) ccons ::= AS generated */ + -2, /* (366) conslist_opt ::= COMMA conslist */ + -3, /* (367) conslist ::= conslist tconscomma tcons */ + -1, /* (368) conslist ::= tcons */ + 0, /* (369) tconscomma ::= */ + -1, /* (370) defer_subclause_opt ::= defer_subclause */ + -1, /* (371) resolvetype ::= raisetype */ + -1, /* (372) selectnowith ::= oneselect */ + -1, /* (373) oneselect ::= values */ + -2, /* (374) sclp ::= selcollist COMMA */ + -1, /* (375) as ::= ID|STRING */ + -1, /* (376) indexed_opt ::= indexed_by */ + 0, /* (377) returning ::= */ + -1, /* (378) expr ::= term */ + -1, /* (379) likeop ::= LIKE_KW|MATCH */ + -1, /* (380) case_operand ::= expr */ + -1, /* (381) exprlist ::= nexprlist */ + -1, /* (382) nmnum ::= plus_num */ + -1, /* (383) nmnum ::= nm */ + -1, /* (384) nmnum ::= ON */ + -1, /* (385) nmnum ::= DELETE */ + -1, /* (386) nmnum ::= DEFAULT */ + -1, /* (387) plus_num ::= INTEGER|FLOAT */ + 0, /* (388) foreach_clause ::= */ + -3, /* (389) foreach_clause ::= FOR EACH ROW */ + -1, /* (390) trnm ::= nm */ + 0, /* (391) tridxby ::= */ + -1, /* (392) database_kw_opt ::= DATABASE */ + 0, /* (393) database_kw_opt ::= */ + 0, /* (394) kwcolumn_opt ::= */ + -1, /* (395) kwcolumn_opt ::= COLUMNKW */ + -1, /* (396) vtabarglist ::= vtabarg */ + -3, /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (398) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (399) anylist ::= */ + -4, /* (400) anylist ::= anylist LP anylist RP */ + -2, /* (401) anylist ::= anylist ANY */ + 0, /* (402) with ::= */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -170378,7 +171315,7 @@ static YYACTIONTYPE yy_reduce( case 5: /* transtype ::= DEFERRED */ case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); - case 323: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==323); + case 321: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==321); {yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/} break; case 8: /* cmd ::= COMMIT|END trans_opt */ @@ -170415,7 +171352,7 @@ static YYACTIONTYPE yy_reduce( case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72); case 81: /* ifexists ::= */ yytestcase(yyruleno==81); case 98: /* distinct ::= */ yytestcase(yyruleno==98); - case 244: /* collate ::= */ yytestcase(yyruleno==244); + case 242: /* collate ::= */ yytestcase(yyruleno==242); {yymsp[1].minor.yy394 = 0;} break; case 16: /* ifnotexists ::= IF NOT EXISTS */ @@ -170599,9 +171536,9 @@ static YYACTIONTYPE yy_reduce( break; case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); - case 216: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==216); - case 219: /* in_op ::= NOT IN */ yytestcase(yyruleno==219); - case 245: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==245); + case 215: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==215); + case 218: /* in_op ::= NOT IN */ yytestcase(yyruleno==218); + case 243: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==243); {yymsp[-1].minor.yy394 = 1;} break; case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ @@ -170751,9 +171688,9 @@ static YYACTIONTYPE yy_reduce( case 99: /* sclp ::= */ case 132: /* orderby_opt ::= */ yytestcase(yyruleno==132); case 142: /* groupby_opt ::= */ yytestcase(yyruleno==142); - case 232: /* exprlist ::= */ yytestcase(yyruleno==232); - case 235: /* paren_exprlist ::= */ yytestcase(yyruleno==235); - case 240: /* eidlist_opt ::= */ yytestcase(yyruleno==240); + case 230: /* exprlist ::= */ yytestcase(yyruleno==230); + case 233: /* paren_exprlist ::= */ yytestcase(yyruleno==233); + case 238: /* eidlist_opt ::= */ yytestcase(yyruleno==238); {yymsp[1].minor.yy322 = 0;} break; case 100: /* selcollist ::= sclp scanpt expr scanpt as */ @@ -170779,8 +171716,8 @@ static YYACTIONTYPE yy_reduce( break; case 103: /* as ::= AS nm */ case 115: /* dbnm ::= DOT nm */ yytestcase(yyruleno==115); - case 256: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==256); - case 257: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==257); + case 254: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==254); + case 255: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==255); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; case 105: /* from ::= */ @@ -170824,7 +171761,7 @@ static YYACTIONTYPE yy_reduce( { if( yymsp[-5].minor.yy131==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy561.pOn==0 && yymsp[0].minor.yy561.pUsing==0 ){ yymsp[-5].minor.yy131 = yymsp[-3].minor.yy131; - }else if( yymsp[-3].minor.yy131->nSrc==1 ){ + }else if( ALWAYS(yymsp[-3].minor.yy131!=0) && yymsp[-3].minor.yy131->nSrc==1 ){ yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561); if( yymsp[-5].minor.yy131 ){ SrcItem *pNew = &yymsp[-5].minor.yy131->a[yymsp[-5].minor.yy131->nSrc-1]; @@ -170952,16 +171889,16 @@ static YYACTIONTYPE yy_reduce( case 146: /* limit_opt ::= */ yytestcase(yyruleno==146); case 151: /* where_opt ::= */ yytestcase(yyruleno==151); case 153: /* where_opt_ret ::= */ yytestcase(yyruleno==153); - case 229: /* case_else ::= */ yytestcase(yyruleno==229); - case 231: /* case_operand ::= */ yytestcase(yyruleno==231); - case 250: /* vinto ::= */ yytestcase(yyruleno==250); + case 228: /* case_else ::= */ yytestcase(yyruleno==228); + case 229: /* case_operand ::= */ yytestcase(yyruleno==229); + case 248: /* vinto ::= */ yytestcase(yyruleno==248); {yymsp[1].minor.yy528 = 0;} break; case 145: /* having_opt ::= HAVING expr */ case 152: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==152); case 154: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==154); - case 228: /* case_else ::= ELSE expr */ yytestcase(yyruleno==228); - case 249: /* vinto ::= INTO expr */ yytestcase(yyruleno==249); + case 227: /* case_else ::= ELSE expr */ yytestcase(yyruleno==227); + case 247: /* vinto ::= INTO expr */ yytestcase(yyruleno==247); {yymsp[-1].minor.yy528 = yymsp[0].minor.yy528;} break; case 147: /* limit_opt ::= LIMIT expr */ @@ -171073,11 +172010,10 @@ static YYACTIONTYPE yy_reduce( case 177: /* expr ::= LP expr RP */ {yymsp[-2].minor.yy528 = yymsp[-1].minor.yy528;} break; - case 178: /* expr ::= ID|INDEXED */ - case 179: /* expr ::= JOIN_KW */ yytestcase(yyruleno==179); + case 178: /* expr ::= ID|INDEXED|JOIN_KW */ {yymsp[0].minor.yy528=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 180: /* expr ::= nm DOT nm */ + case 179: /* expr ::= nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); @@ -171085,7 +172021,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy528 = yylhsminor.yy528; break; - case 181: /* expr ::= nm DOT nm DOT nm */ + case 180: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-4].minor.yy0); Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); @@ -171098,18 +172034,18 @@ static YYACTIONTYPE yy_reduce( } yymsp[-4].minor.yy528 = yylhsminor.yy528; break; - case 182: /* term ::= NULL|FLOAT|BLOB */ - case 183: /* term ::= STRING */ yytestcase(yyruleno==183); + case 181: /* term ::= NULL|FLOAT|BLOB */ + case 182: /* term ::= STRING */ yytestcase(yyruleno==182); {yymsp[0].minor.yy528=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 184: /* term ::= INTEGER */ + case 183: /* term ::= INTEGER */ { yylhsminor.yy528 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); if( yylhsminor.yy528 ) yylhsminor.yy528->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); } yymsp[0].minor.yy528 = yylhsminor.yy528; break; - case 185: /* expr ::= VARIABLE */ + case 184: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ u32 n = yymsp[0].minor.yy0.n; @@ -171131,50 +172067,50 @@ static YYACTIONTYPE yy_reduce( } } break; - case 186: /* expr ::= expr COLLATE ID|STRING */ + case 185: /* expr ::= expr COLLATE ID|STRING */ { yymsp[-2].minor.yy528 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy528, &yymsp[0].minor.yy0, 1); } break; - case 187: /* expr ::= CAST LP expr AS typetoken RP */ + case 186: /* expr ::= CAST LP expr AS typetoken RP */ { yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy528, yymsp[-3].minor.yy528, 0); } break; - case 188: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 187: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy394); } yymsp[-4].minor.yy528 = yylhsminor.yy528; break; - case 189: /* expr ::= ID|INDEXED LP STAR RP */ + case 188: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } yymsp[-3].minor.yy528 = yylhsminor.yy528; break; - case 190: /* expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ + case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy322, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy394); sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); } yymsp[-5].minor.yy528 = yylhsminor.yy528; break; - case 191: /* expr ::= ID|INDEXED LP STAR RP filter_over */ + case 190: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41); } yymsp[-4].minor.yy528 = yylhsminor.yy528; break; - case 192: /* term ::= CTIME_KW */ + case 191: /* term ::= CTIME_KW */ { yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } yymsp[0].minor.yy528 = yylhsminor.yy528; break; - case 193: /* expr ::= LP nexprlist COMMA expr RP */ + case 192: /* expr ::= LP nexprlist COMMA expr RP */ { ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528); yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); @@ -171188,22 +172124,22 @@ static YYACTIONTYPE yy_reduce( } } break; - case 194: /* expr ::= expr AND expr */ + case 193: /* expr ::= expr AND expr */ {yymsp[-2].minor.yy528=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} break; - case 195: /* expr ::= expr OR expr */ - case 196: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==196); - case 197: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==197); - case 198: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==198); - case 199: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==199); - case 200: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==200); - case 201: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==201); + case 194: /* expr ::= expr OR expr */ + case 195: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==195); + case 196: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==196); + case 197: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==197); + case 198: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==198); + case 199: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==200); {yymsp[-2].minor.yy528=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);} break; - case 202: /* likeop ::= NOT LIKE_KW|MATCH */ + case 201: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} break; - case 203: /* expr ::= expr likeop expr */ + case 202: /* expr ::= expr likeop expr */ { ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; @@ -171215,7 +172151,7 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-2].minor.yy528 ) yymsp[-2].minor.yy528->flags |= EP_InfixFunc; } break; - case 204: /* expr ::= expr likeop expr ESCAPE expr */ + case 203: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; @@ -171228,47 +172164,47 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-4].minor.yy528 ) yymsp[-4].minor.yy528->flags |= EP_InfixFunc; } break; - case 205: /* expr ::= expr ISNULL|NOTNULL */ + case 204: /* expr ::= expr ISNULL|NOTNULL */ {yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy528,0);} break; - case 206: /* expr ::= expr NOT NULL */ + case 205: /* expr ::= expr NOT NULL */ {yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy528,0);} break; - case 207: /* expr ::= expr IS expr */ + case 206: /* expr ::= expr IS expr */ { yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-2].minor.yy528, TK_ISNULL); } break; - case 208: /* expr ::= expr IS NOT expr */ + case 207: /* expr ::= expr IS NOT expr */ { yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-3].minor.yy528, TK_NOTNULL); } break; - case 209: /* expr ::= expr IS NOT DISTINCT FROM expr */ + case 208: /* expr ::= expr IS NOT DISTINCT FROM expr */ { yymsp[-5].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-5].minor.yy528, TK_ISNULL); } break; - case 210: /* expr ::= expr IS DISTINCT FROM expr */ + case 209: /* expr ::= expr IS DISTINCT FROM expr */ { yymsp[-4].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy528,yymsp[0].minor.yy528); binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-4].minor.yy528, TK_NOTNULL); } break; - case 211: /* expr ::= NOT expr */ - case 212: /* expr ::= BITNOT expr */ yytestcase(yyruleno==212); + case 210: /* expr ::= NOT expr */ + case 211: /* expr ::= BITNOT expr */ yytestcase(yyruleno==211); {yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy528, 0);/*A-overwrites-B*/} break; - case 213: /* expr ::= PLUS|MINUS expr */ + case 212: /* expr ::= PLUS|MINUS expr */ { yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy528, 0); /*A-overwrites-B*/ } break; - case 214: /* expr ::= expr PTR expr */ + case 213: /* expr ::= expr PTR expr */ { ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy528); pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy528); @@ -171276,11 +172212,11 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy528 = yylhsminor.yy528; break; - case 215: /* between_op ::= BETWEEN */ - case 218: /* in_op ::= IN */ yytestcase(yyruleno==218); + case 214: /* between_op ::= BETWEEN */ + case 217: /* in_op ::= IN */ yytestcase(yyruleno==217); {yymsp[0].minor.yy394 = 0;} break; - case 217: /* expr ::= expr between_op expr AND expr */ + case 216: /* expr ::= expr between_op expr AND expr */ { ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528); @@ -171293,7 +172229,7 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); } break; - case 220: /* expr ::= expr in_op LP exprlist RP */ + case 219: /* expr ::= expr in_op LP exprlist RP */ { if( yymsp[-1].minor.yy322==0 ){ /* Expressions of the form @@ -171339,20 +172275,20 @@ static YYACTIONTYPE yy_reduce( } } break; - case 221: /* expr ::= LP select RP */ + case 220: /* expr ::= LP select RP */ { yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy528, yymsp[-1].minor.yy47); } break; - case 222: /* expr ::= expr in_op LP select RP */ + case 221: /* expr ::= expr in_op LP select RP */ { yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0); sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, yymsp[-1].minor.yy47); if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); } break; - case 223: /* expr ::= expr in_op nm dbnm paren_exprlist */ + case 222: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); @@ -171362,14 +172298,14 @@ static YYACTIONTYPE yy_reduce( if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0); } break; - case 224: /* expr ::= EXISTS LP select RP */ + case 223: /* expr ::= EXISTS LP select RP */ { Expr *p; p = yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy47); } break; - case 225: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 224: /* expr ::= CASE case_operand case_exprlist case_else END */ { yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy528, 0); if( yymsp[-4].minor.yy528 ){ @@ -171381,32 +172317,29 @@ static YYACTIONTYPE yy_reduce( } } break; - case 226: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 225: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy528); yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[0].minor.yy528); } break; - case 227: /* case_exprlist ::= WHEN expr THEN expr */ + case 226: /* case_exprlist ::= WHEN expr THEN expr */ { yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528); yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, yymsp[0].minor.yy528); } break; - case 230: /* case_operand ::= expr */ -{yymsp[0].minor.yy528 = yymsp[0].minor.yy528; /*A-overwrites-X*/} - break; - case 233: /* nexprlist ::= nexprlist COMMA expr */ + case 231: /* nexprlist ::= nexprlist COMMA expr */ {yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy528);} break; - case 234: /* nexprlist ::= expr */ + case 232: /* nexprlist ::= expr */ {yymsp[0].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy528); /*A-overwrites-Y*/} break; - case 236: /* paren_exprlist ::= LP exprlist RP */ - case 241: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==241); + case 234: /* paren_exprlist ::= LP exprlist RP */ + case 239: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==239); {yymsp[-2].minor.yy322 = yymsp[-1].minor.yy322;} break; - case 237: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 235: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy322, yymsp[-10].minor.yy394, @@ -171416,48 +172349,48 @@ static YYACTIONTYPE yy_reduce( } } break; - case 238: /* uniqueflag ::= UNIQUE */ - case 280: /* raisetype ::= ABORT */ yytestcase(yyruleno==280); + case 236: /* uniqueflag ::= UNIQUE */ + case 278: /* raisetype ::= ABORT */ yytestcase(yyruleno==278); {yymsp[0].minor.yy394 = OE_Abort;} break; - case 239: /* uniqueflag ::= */ + case 237: /* uniqueflag ::= */ {yymsp[1].minor.yy394 = OE_None;} break; - case 242: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 240: /* eidlist ::= eidlist COMMA nm collate sortorder */ { yymsp[-4].minor.yy322 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); } break; - case 243: /* eidlist ::= nm collate sortorder */ + case 241: /* eidlist ::= nm collate sortorder */ { yymsp[-2].minor.yy322 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); /*A-overwrites-Y*/ } break; - case 246: /* cmd ::= DROP INDEX ifexists fullname */ + case 244: /* cmd ::= DROP INDEX ifexists fullname */ {sqlite3DropIndex(pParse, yymsp[0].minor.yy131, yymsp[-1].minor.yy394);} break; - case 247: /* cmd ::= VACUUM vinto */ + case 245: /* cmd ::= VACUUM vinto */ {sqlite3Vacuum(pParse,0,yymsp[0].minor.yy528);} break; - case 248: /* cmd ::= VACUUM nm vinto */ + case 246: /* cmd ::= VACUUM nm vinto */ {sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy528);} break; - case 251: /* cmd ::= PRAGMA nm dbnm */ + case 249: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 250: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 251: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 252: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 253: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 258: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 256: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; @@ -171465,50 +172398,50 @@ static YYACTIONTYPE yy_reduce( sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy33, &all); } break; - case 259: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 257: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy394, yymsp[-4].minor.yy180.a, yymsp[-4].minor.yy180.b, yymsp[-2].minor.yy131, yymsp[0].minor.yy528, yymsp[-10].minor.yy394, yymsp[-8].minor.yy394); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 260: /* trigger_time ::= BEFORE|AFTER */ + case 258: /* trigger_time ::= BEFORE|AFTER */ { yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 261: /* trigger_time ::= INSTEAD OF */ + case 259: /* trigger_time ::= INSTEAD OF */ { yymsp[-1].minor.yy394 = TK_INSTEAD;} break; - case 262: /* trigger_time ::= */ + case 260: /* trigger_time ::= */ { yymsp[1].minor.yy394 = TK_BEFORE; } break; - case 263: /* trigger_event ::= DELETE|INSERT */ - case 264: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==264); + case 261: /* trigger_event ::= DELETE|INSERT */ + case 262: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==262); {yymsp[0].minor.yy180.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy180.b = 0;} break; - case 265: /* trigger_event ::= UPDATE OF idlist */ + case 263: /* trigger_event ::= UPDATE OF idlist */ {yymsp[-2].minor.yy180.a = TK_UPDATE; yymsp[-2].minor.yy180.b = yymsp[0].minor.yy254;} break; - case 266: /* when_clause ::= */ - case 285: /* key_opt ::= */ yytestcase(yyruleno==285); + case 264: /* when_clause ::= */ + case 283: /* key_opt ::= */ yytestcase(yyruleno==283); { yymsp[1].minor.yy528 = 0; } break; - case 267: /* when_clause ::= WHEN expr */ - case 286: /* key_opt ::= KEY expr */ yytestcase(yyruleno==286); + case 265: /* when_clause ::= WHEN expr */ + case 284: /* key_opt ::= KEY expr */ yytestcase(yyruleno==284); { yymsp[-1].minor.yy528 = yymsp[0].minor.yy528; } break; - case 268: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 266: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { assert( yymsp[-2].minor.yy33!=0 ); yymsp[-2].minor.yy33->pLast->pNext = yymsp[-1].minor.yy33; yymsp[-2].minor.yy33->pLast = yymsp[-1].minor.yy33; } break; - case 269: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 267: /* trigger_cmd_list ::= trigger_cmd SEMI */ { assert( yymsp[-1].minor.yy33!=0 ); yymsp[-1].minor.yy33->pLast = yymsp[-1].minor.yy33; } break; - case 270: /* trnm ::= nm DOT nm */ + case 268: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -171516,39 +172449,39 @@ static YYACTIONTYPE yy_reduce( "statements within triggers"); } break; - case 271: /* tridxby ::= INDEXED BY nm */ + case 269: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 272: /* tridxby ::= NOT INDEXED */ + case 270: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 273: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + case 271: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ {yylhsminor.yy33 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy131, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528, yymsp[-7].minor.yy394, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy522);} yymsp[-8].minor.yy33 = yylhsminor.yy33; break; - case 274: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 272: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { yylhsminor.yy33 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy254,yymsp[-2].minor.yy47,yymsp[-6].minor.yy394,yymsp[-1].minor.yy444,yymsp[-7].minor.yy522,yymsp[0].minor.yy522);/*yylhsminor.yy33-overwrites-yymsp[-6].minor.yy394*/ } yymsp[-7].minor.yy33 = yylhsminor.yy33; break; - case 275: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + case 273: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ {yylhsminor.yy33 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy528, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy522);} yymsp[-5].minor.yy33 = yylhsminor.yy33; break; - case 276: /* trigger_cmd ::= scanpt select scanpt */ + case 274: /* trigger_cmd ::= scanpt select scanpt */ {yylhsminor.yy33 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy47, yymsp[-2].minor.yy522, yymsp[0].minor.yy522); /*yylhsminor.yy33-overwrites-yymsp[-1].minor.yy47*/} yymsp[-2].minor.yy33 = yylhsminor.yy33; break; - case 277: /* expr ::= RAISE LP IGNORE RP */ + case 275: /* expr ::= RAISE LP IGNORE RP */ { yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); if( yymsp[-3].minor.yy528 ){ @@ -171556,7 +172489,7 @@ static YYACTIONTYPE yy_reduce( } } break; - case 278: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 276: /* expr ::= RAISE LP raisetype COMMA nm RP */ { yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); if( yymsp[-5].minor.yy528 ) { @@ -171564,118 +172497,118 @@ static YYACTIONTYPE yy_reduce( } } break; - case 279: /* raisetype ::= ROLLBACK */ + case 277: /* raisetype ::= ROLLBACK */ {yymsp[0].minor.yy394 = OE_Rollback;} break; - case 281: /* raisetype ::= FAIL */ + case 279: /* raisetype ::= FAIL */ {yymsp[0].minor.yy394 = OE_Fail;} break; - case 282: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 280: /* cmd ::= DROP TRIGGER ifexists fullname */ { sqlite3DropTrigger(pParse,yymsp[0].minor.yy131,yymsp[-1].minor.yy394); } break; - case 283: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 281: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { sqlite3Attach(pParse, yymsp[-3].minor.yy528, yymsp[-1].minor.yy528, yymsp[0].minor.yy528); } break; - case 284: /* cmd ::= DETACH database_kw_opt expr */ + case 282: /* cmd ::= DETACH database_kw_opt expr */ { sqlite3Detach(pParse, yymsp[0].minor.yy528); } break; - case 287: /* cmd ::= REINDEX */ + case 285: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 288: /* cmd ::= REINDEX nm dbnm */ + case 286: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 289: /* cmd ::= ANALYZE */ + case 287: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 290: /* cmd ::= ANALYZE nm dbnm */ + case 288: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 291: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 289: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy131,&yymsp[0].minor.yy0); } break; - case 292: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 290: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 293: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + case 291: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ { sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy131, &yymsp[0].minor.yy0); } break; - case 294: /* add_column_fullname ::= fullname */ + case 292: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy131); } break; - case 295: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + case 293: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ { sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy131, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 296: /* cmd ::= create_vtab */ + case 294: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 297: /* cmd ::= create_vtab LP vtabarglist RP */ + case 295: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 298: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 296: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy394); } break; - case 299: /* vtabarg ::= */ + case 297: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 300: /* vtabargtoken ::= ANY */ - case 301: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==301); - case 302: /* lp ::= LP */ yytestcase(yyruleno==302); + case 298: /* vtabargtoken ::= ANY */ + case 299: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==299); + case 300: /* lp ::= LP */ yytestcase(yyruleno==300); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 303: /* with ::= WITH wqlist */ - case 304: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==304); + case 301: /* with ::= WITH wqlist */ + case 302: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==302); { sqlite3WithPush(pParse, yymsp[0].minor.yy521, 1); } break; - case 305: /* wqas ::= AS */ + case 303: /* wqas ::= AS */ {yymsp[0].minor.yy516 = M10d_Any;} break; - case 306: /* wqas ::= AS MATERIALIZED */ + case 304: /* wqas ::= AS MATERIALIZED */ {yymsp[-1].minor.yy516 = M10d_Yes;} break; - case 307: /* wqas ::= AS NOT MATERIALIZED */ + case 305: /* wqas ::= AS NOT MATERIALIZED */ {yymsp[-2].minor.yy516 = M10d_No;} break; - case 308: /* wqitem ::= nm eidlist_opt wqas LP select RP */ + case 306: /* wqitem ::= nm eidlist_opt wqas LP select RP */ { yymsp[-5].minor.yy385 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy322, yymsp[-1].minor.yy47, yymsp[-3].minor.yy516); /*A-overwrites-X*/ } break; - case 309: /* wqlist ::= wqitem */ + case 307: /* wqlist ::= wqitem */ { yymsp[0].minor.yy521 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy385); /*A-overwrites-X*/ } break; - case 310: /* wqlist ::= wqlist COMMA wqitem */ + case 308: /* wqlist ::= wqlist COMMA wqitem */ { yymsp[-2].minor.yy521 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy521, yymsp[0].minor.yy385); } break; - case 311: /* windowdefn_list ::= windowdefn */ + case 309: /* windowdefn_list ::= windowdefn */ { yylhsminor.yy41 = yymsp[0].minor.yy41; } yymsp[0].minor.yy41 = yylhsminor.yy41; break; - case 312: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ + case 310: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ { assert( yymsp[0].minor.yy41!=0 ); sqlite3WindowChain(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy41); @@ -171684,7 +172617,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-2].minor.yy41 = yylhsminor.yy41; break; - case 313: /* windowdefn ::= nm AS LP window RP */ + case 311: /* windowdefn ::= nm AS LP window RP */ { if( ALWAYS(yymsp[-1].minor.yy41) ){ yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); @@ -171693,90 +172626,90 @@ static YYACTIONTYPE yy_reduce( } yymsp[-4].minor.yy41 = yylhsminor.yy41; break; - case 314: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + case 312: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ { yymsp[-4].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, 0); } break; - case 315: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + case 313: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ { yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, &yymsp[-5].minor.yy0); } yymsp[-5].minor.yy41 = yylhsminor.yy41; break; - case 316: /* window ::= ORDER BY sortlist frame_opt */ + case 314: /* window ::= ORDER BY sortlist frame_opt */ { yymsp[-3].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, 0); } break; - case 317: /* window ::= nm ORDER BY sortlist frame_opt */ + case 315: /* window ::= nm ORDER BY sortlist frame_opt */ { yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0); } yymsp[-4].minor.yy41 = yylhsminor.yy41; break; - case 318: /* window ::= frame_opt */ - case 337: /* filter_over ::= over_clause */ yytestcase(yyruleno==337); + case 316: /* window ::= frame_opt */ + case 335: /* filter_over ::= over_clause */ yytestcase(yyruleno==335); { yylhsminor.yy41 = yymsp[0].minor.yy41; } yymsp[0].minor.yy41 = yylhsminor.yy41; break; - case 319: /* window ::= nm frame_opt */ + case 317: /* window ::= nm frame_opt */ { yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, 0, &yymsp[-1].minor.yy0); } yymsp[-1].minor.yy41 = yylhsminor.yy41; break; - case 320: /* frame_opt ::= */ + case 318: /* frame_opt ::= */ { yymsp[1].minor.yy41 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); } break; - case 321: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + case 319: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ { yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy394, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy516); } yymsp[-2].minor.yy41 = yylhsminor.yy41; break; - case 322: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + case 320: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ { yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy394, yymsp[-3].minor.yy595.eType, yymsp[-3].minor.yy595.pExpr, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, yymsp[0].minor.yy516); } yymsp[-5].minor.yy41 = yylhsminor.yy41; break; - case 324: /* frame_bound_s ::= frame_bound */ - case 326: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==326); + case 322: /* frame_bound_s ::= frame_bound */ + case 324: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==324); {yylhsminor.yy595 = yymsp[0].minor.yy595;} yymsp[0].minor.yy595 = yylhsminor.yy595; break; - case 325: /* frame_bound_s ::= UNBOUNDED PRECEDING */ - case 327: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==327); - case 329: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==329); + case 323: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 325: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==325); + case 327: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==327); {yylhsminor.yy595.eType = yymsp[-1].major; yylhsminor.yy595.pExpr = 0;} yymsp[-1].minor.yy595 = yylhsminor.yy595; break; - case 328: /* frame_bound ::= expr PRECEDING|FOLLOWING */ + case 326: /* frame_bound ::= expr PRECEDING|FOLLOWING */ {yylhsminor.yy595.eType = yymsp[0].major; yylhsminor.yy595.pExpr = yymsp[-1].minor.yy528;} yymsp[-1].minor.yy595 = yylhsminor.yy595; break; - case 330: /* frame_exclude_opt ::= */ + case 328: /* frame_exclude_opt ::= */ {yymsp[1].minor.yy516 = 0;} break; - case 331: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ + case 329: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ {yymsp[-1].minor.yy516 = yymsp[0].minor.yy516;} break; - case 332: /* frame_exclude ::= NO OTHERS */ - case 333: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==333); + case 330: /* frame_exclude ::= NO OTHERS */ + case 331: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==331); {yymsp[-1].minor.yy516 = yymsp[-1].major; /*A-overwrites-X*/} break; - case 334: /* frame_exclude ::= GROUP|TIES */ + case 332: /* frame_exclude ::= GROUP|TIES */ {yymsp[0].minor.yy516 = yymsp[0].major; /*A-overwrites-X*/} break; - case 335: /* window_clause ::= WINDOW windowdefn_list */ + case 333: /* window_clause ::= WINDOW windowdefn_list */ { yymsp[-1].minor.yy41 = yymsp[0].minor.yy41; } break; - case 336: /* filter_over ::= filter_clause over_clause */ + case 334: /* filter_over ::= filter_clause over_clause */ { if( yymsp[0].minor.yy41 ){ yymsp[0].minor.yy41->pFilter = yymsp[-1].minor.yy528; @@ -171787,7 +172720,7 @@ static YYACTIONTYPE yy_reduce( } yymsp[-1].minor.yy41 = yylhsminor.yy41; break; - case 338: /* filter_over ::= filter_clause */ + case 336: /* filter_over ::= filter_clause */ { yylhsminor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); if( yylhsminor.yy41 ){ @@ -171799,13 +172732,13 @@ static YYACTIONTYPE yy_reduce( } yymsp[0].minor.yy41 = yylhsminor.yy41; break; - case 339: /* over_clause ::= OVER LP window RP */ + case 337: /* over_clause ::= OVER LP window RP */ { yymsp[-3].minor.yy41 = yymsp[-1].minor.yy41; assert( yymsp[-3].minor.yy41!=0 ); } break; - case 340: /* over_clause ::= OVER nm */ + case 338: /* over_clause ::= OVER nm */ { yymsp[-1].minor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); if( yymsp[-1].minor.yy41 ){ @@ -171813,73 +172746,73 @@ static YYACTIONTYPE yy_reduce( } } break; - case 341: /* filter_clause ::= FILTER LP WHERE expr RP */ + case 339: /* filter_clause ::= FILTER LP WHERE expr RP */ { yymsp[-4].minor.yy528 = yymsp[-1].minor.yy528; } break; default: - /* (342) input ::= cmdlist */ yytestcase(yyruleno==342); - /* (343) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==343); - /* (344) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=344); - /* (345) ecmd ::= SEMI */ yytestcase(yyruleno==345); - /* (346) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==346); - /* (347) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=347); - /* (348) trans_opt ::= */ yytestcase(yyruleno==348); - /* (349) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==349); - /* (350) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==350); - /* (351) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==351); - /* (352) savepoint_opt ::= */ yytestcase(yyruleno==352); - /* (353) cmd ::= create_table create_table_args */ yytestcase(yyruleno==353); - /* (354) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=354); - /* (355) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==355); - /* (356) columnlist ::= columnname carglist */ yytestcase(yyruleno==356); - /* (357) nm ::= ID|INDEXED */ yytestcase(yyruleno==357); - /* (358) nm ::= STRING */ yytestcase(yyruleno==358); - /* (359) nm ::= JOIN_KW */ yytestcase(yyruleno==359); - /* (360) typetoken ::= typename */ yytestcase(yyruleno==360); - /* (361) typename ::= ID|STRING */ yytestcase(yyruleno==361); - /* (362) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=362); - /* (363) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=363); - /* (364) carglist ::= carglist ccons */ yytestcase(yyruleno==364); - /* (365) carglist ::= */ yytestcase(yyruleno==365); - /* (366) ccons ::= NULL onconf */ yytestcase(yyruleno==366); - /* (367) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==367); - /* (368) ccons ::= AS generated */ yytestcase(yyruleno==368); - /* (369) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==369); - /* (370) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==370); - /* (371) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=371); - /* (372) tconscomma ::= */ yytestcase(yyruleno==372); - /* (373) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=373); - /* (374) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=374); - /* (375) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=375); - /* (376) oneselect ::= values */ yytestcase(yyruleno==376); - /* (377) sclp ::= selcollist COMMA */ yytestcase(yyruleno==377); - /* (378) as ::= ID|STRING */ yytestcase(yyruleno==378); - /* (379) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=379); - /* (380) returning ::= */ yytestcase(yyruleno==380); - /* (381) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=381); - /* (382) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==382); - /* (383) exprlist ::= nexprlist */ yytestcase(yyruleno==383); - /* (384) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=384); - /* (385) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=385); - /* (386) nmnum ::= ON */ yytestcase(yyruleno==386); - /* (387) nmnum ::= DELETE */ yytestcase(yyruleno==387); - /* (388) nmnum ::= DEFAULT */ yytestcase(yyruleno==388); - /* (389) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==389); - /* (390) foreach_clause ::= */ yytestcase(yyruleno==390); - /* (391) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==391); - /* (392) trnm ::= nm */ yytestcase(yyruleno==392); - /* (393) tridxby ::= */ yytestcase(yyruleno==393); - /* (394) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==394); - /* (395) database_kw_opt ::= */ yytestcase(yyruleno==395); - /* (396) kwcolumn_opt ::= */ yytestcase(yyruleno==396); - /* (397) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==397); - /* (398) vtabarglist ::= vtabarg */ yytestcase(yyruleno==398); - /* (399) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==399); - /* (400) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==400); - /* (401) anylist ::= */ yytestcase(yyruleno==401); - /* (402) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==402); - /* (403) anylist ::= anylist ANY */ yytestcase(yyruleno==403); - /* (404) with ::= */ yytestcase(yyruleno==404); + /* (340) input ::= cmdlist */ yytestcase(yyruleno==340); + /* (341) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==341); + /* (342) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=342); + /* (343) ecmd ::= SEMI */ yytestcase(yyruleno==343); + /* (344) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==344); + /* (345) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=345); + /* (346) trans_opt ::= */ yytestcase(yyruleno==346); + /* (347) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==347); + /* (348) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==348); + /* (349) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==349); + /* (350) savepoint_opt ::= */ yytestcase(yyruleno==350); + /* (351) cmd ::= create_table create_table_args */ yytestcase(yyruleno==351); + /* (352) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=352); + /* (353) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==353); + /* (354) columnlist ::= columnname carglist */ yytestcase(yyruleno==354); + /* (355) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==355); + /* (356) nm ::= STRING */ yytestcase(yyruleno==356); + /* (357) typetoken ::= typename */ yytestcase(yyruleno==357); + /* (358) typename ::= ID|STRING */ yytestcase(yyruleno==358); + /* (359) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=359); + /* (360) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=360); + /* (361) carglist ::= carglist ccons */ yytestcase(yyruleno==361); + /* (362) carglist ::= */ yytestcase(yyruleno==362); + /* (363) ccons ::= NULL onconf */ yytestcase(yyruleno==363); + /* (364) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==364); + /* (365) ccons ::= AS generated */ yytestcase(yyruleno==365); + /* (366) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==366); + /* (367) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==367); + /* (368) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=368); + /* (369) tconscomma ::= */ yytestcase(yyruleno==369); + /* (370) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=370); + /* (371) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=371); + /* (372) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=372); + /* (373) oneselect ::= values */ yytestcase(yyruleno==373); + /* (374) sclp ::= selcollist COMMA */ yytestcase(yyruleno==374); + /* (375) as ::= ID|STRING */ yytestcase(yyruleno==375); + /* (376) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=376); + /* (377) returning ::= */ yytestcase(yyruleno==377); + /* (378) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=378); + /* (379) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==379); + /* (380) case_operand ::= expr */ yytestcase(yyruleno==380); + /* (381) exprlist ::= nexprlist */ yytestcase(yyruleno==381); + /* (382) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=382); + /* (383) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=383); + /* (384) nmnum ::= ON */ yytestcase(yyruleno==384); + /* (385) nmnum ::= DELETE */ yytestcase(yyruleno==385); + /* (386) nmnum ::= DEFAULT */ yytestcase(yyruleno==386); + /* (387) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==387); + /* (388) foreach_clause ::= */ yytestcase(yyruleno==388); + /* (389) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==389); + /* (390) trnm ::= nm */ yytestcase(yyruleno==390); + /* (391) tridxby ::= */ yytestcase(yyruleno==391); + /* (392) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==392); + /* (393) database_kw_opt ::= */ yytestcase(yyruleno==393); + /* (394) kwcolumn_opt ::= */ yytestcase(yyruleno==394); + /* (395) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==395); + /* (396) vtabarglist ::= vtabarg */ yytestcase(yyruleno==396); + /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==397); + /* (398) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==398); + /* (399) anylist ::= */ yytestcase(yyruleno==399); + /* (400) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==400); + /* (401) anylist ::= anylist ANY */ yytestcase(yyruleno==401); + /* (402) with ::= */ yytestcase(yyruleno==402); break; /********** End reduce actions ************************************************/ }; @@ -172455,7 +173388,7 @@ static const unsigned char aKWHash[127] = { /* aKWNext[] forms the hash collision chain. If aKWHash[i]==0 ** then the i-th keyword has no more hash collisions. Otherwise, ** the next keyword with the same hash is aKWHash[i]-1. */ -static const unsigned char aKWNext[147] = { +static const unsigned char aKWNext[148] = {0, 0, 0, 0, 0, 4, 0, 43, 0, 0, 106, 114, 0, 0, 0, 2, 0, 0, 143, 0, 0, 0, 13, 0, 0, 0, 0, 141, 0, 0, 119, 52, 0, 0, 137, 12, 0, 0, 62, 0, @@ -172470,7 +173403,7 @@ static const unsigned char aKWNext[147] = { 102, 0, 0, 87, }; /* aKWLen[i] is the length (in bytes) of the i-th keyword */ -static const unsigned char aKWLen[147] = { +static const unsigned char aKWLen[148] = {0, 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 7, 6, 9, 4, 2, 6, 5, 9, 9, 4, 7, 3, 2, 4, @@ -172486,7 +173419,7 @@ static const unsigned char aKWLen[147] = { }; /* aKWOffset[i] is the index into zKWText[] of the start of ** the text for the i-th keyword. */ -static const unsigned short int aKWOffset[147] = { +static const unsigned short int aKWOffset[148] = {0, 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, 86, 90, 90, 94, 99, 101, 105, 111, 119, 123, 123, 123, 126, @@ -172501,7 +173434,7 @@ static const unsigned short int aKWOffset[147] = { 648, 650, 655, 659, }; /* aKWCode[i] is the parser symbol code for the i-th keyword */ -static const unsigned char aKWCode[147] = { +static const unsigned char aKWCode[148] = {0, TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, @@ -172670,7 +173603,7 @@ static int keywordCode(const char *z, int n, int *pType){ const char *zKW; if( n>=2 ){ i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; - for(i=((int)aKWHash[i])-1; i>=0; i=((int)aKWNext[i])-1){ + for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){ if( aKWLen[i]!=n ) continue; zKW = &zKWText[aKWOffset[i]]; #ifdef SQLITE_ASCII @@ -172686,153 +173619,153 @@ static int keywordCode(const char *z, int n, int *pType){ while( j=SQLITE_N_KEYWORD ) return SQLITE_ERROR; + i++; *pzName = zKWText + aKWOffset[i]; *pnName = aKWLen[i]; return SQLITE_OK; @@ -174385,9 +175319,21 @@ SQLITE_API int sqlite3_config(int op, ...){ va_list ap; int rc = SQLITE_OK; - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + /* sqlite3_config() normally returns SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. Except, a few selected opcodes + ** are allowed. + */ + if( sqlite3GlobalConfig.isInit ){ + static const u64 mAnytimeConfigOption = 0 + | MASKBIT64( SQLITE_CONFIG_LOG ) + | MASKBIT64( SQLITE_CONFIG_PCACHE_HDRSZ ) + ; + if( op<0 || op>63 || (MASKBIT64(op) & mAnytimeConfigOption)==0 ){ + return SQLITE_MISUSE_BKPT; + } + testcase( op==SQLITE_CONFIG_LOG ); + testcase( op==SQLITE_CONFIG_PCACHE_HDRSZ ); + } va_start(ap, op); switch( op ){ @@ -174456,6 +175402,7 @@ SQLITE_API int sqlite3_config(int op, ...){ break; } case SQLITE_CONFIG_MEMSTATUS: { + assert( !sqlite3GlobalConfig.isInit ); /* Cannot change at runtime */ /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes ** single argument of type int, interpreted as a boolean, which enables ** or disables the collection of memory allocation statistics. */ @@ -174579,8 +175526,10 @@ SQLITE_API int sqlite3_config(int op, ...){ ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); */ typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + LOGFUNC_t xLog = va_arg(ap, LOGFUNC_t); + void *pLogArg = va_arg(ap, void*); + AtomicStore(&sqlite3GlobalConfig.xLog, xLog); + AtomicStore(&sqlite3GlobalConfig.pLogArg, pLogArg); break; } @@ -174594,7 +175543,8 @@ SQLITE_API int sqlite3_config(int op, ...){ ** argument of type int. If non-zero, then URI handling is globally ** enabled. If the parameter is zero, then URI handling is globally ** disabled. */ - sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + int bOpenUri = va_arg(ap, int); + AtomicStore(&sqlite3GlobalConfig.bOpenUri, bOpenUri); break; } @@ -174909,6 +175859,8 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, + { SQLITE_DBCONFIG_STMT_SCANSTATUS, SQLITE_StmtScanStatus }, + { SQLITE_DBCONFIG_REVERSE_SCANORDER, SQLITE_ReverseOrder }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -175427,7 +176379,8 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** Return a static string containing the name corresponding to the error code ** specified in the argument. */ -SQLITE_API const char *sqlite3ErrName(int rc){ +#if defined(SQLITE_NEED_ERR_NAME) +SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ const char *zName = 0; int i, origRc = rc; for(i=0; i<2 && zName==0; i++, rc &= 0xff){ @@ -175533,6 +176486,7 @@ SQLITE_API const char *sqlite3ErrName(int rc){ } return zName; } +#endif /* ** Return a static string that describes the kind of error specified in the @@ -176892,9 +177846,9 @@ SQLITE_PRIVATE int sqlite3ParseUri( assert( *pzErrMsg==0 ); - if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ - || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ - && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || AtomicLoad(&sqlite3GlobalConfig.bOpenUri)) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; int eState; /* Parser state when parsing URI */ @@ -177300,6 +178254,9 @@ static int openDatabase( #endif #if defined(SQLITE_DEFAULT_LEGACY_ALTER_TABLE) | SQLITE_LegacyAlter +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) + | SQLITE_StmtScanStatus #endif ; sqlite3HashInit(&db->aCollSeq); @@ -177865,7 +178822,7 @@ SQLITE_API int sqlite3_sleep(int ms){ /* This function works in milliseconds, but the underlying OsSleep() ** API uses microseconds. Hence the 1000's. */ - rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + rc = (sqlite3OsSleep(pVfs, ms<0 ? 0 : 1000*ms)/1000); return rc; } @@ -193069,16 +194026,18 @@ static int fts3MsrBufferData( char *pList, i64 nList ){ - if( nList>pMsr->nBuffer ){ + if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){ char *pNew; - pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, pMsr->nBuffer); + int nNew = nList*2 + FTS3_NODE_PADDING; + pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew); if( !pNew ) return SQLITE_NOMEM; pMsr->aBuffer = pNew; + pMsr->nBuffer = nNew; } assert( nList>0 ); memcpy(pMsr->aBuffer, pList, nList); + memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING); return SQLITE_OK; } @@ -198867,6 +199826,7 @@ static const char * const jsonType[] = { #define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ #define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ #define JNODE_LABEL 0x40 /* Is a label of an object */ +#define JNODE_JSON5 0x80 /* Node contains JSON5 enhancements */ /* A single node of parsed JSON @@ -198893,10 +199853,12 @@ struct JsonParse { JsonNode *aNode; /* Array of nodes containing the parse */ const char *zJson; /* Original JSON string */ u32 *aUp; /* Index of parent of each node */ - u8 oom; /* Set to true if out of memory */ - u8 nErr; /* Number of errors seen */ u16 iDepth; /* Nesting depth */ + u8 nErr; /* Number of errors seen */ + u8 oom; /* Set to true if out of memory */ + u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ int nJson; /* Length of the zJson string in bytes */ + u32 iErr; /* Error location in zJson[] */ u32 iHold; /* Replace cache line with the lowest iHold value */ }; @@ -198904,10 +199866,10 @@ struct JsonParse { ** Maximum nesting depth of JSON for this implementation. ** ** This limit is needed to avoid a stack overflow in the recursive -** descent parser. A depth of 2000 is far deeper than any sane JSON -** should go. +** descent parser. A depth of 1000 is far deeper than any sane JSON +** should go. Historical note: This limit was 2000 prior to version 3.42.0 */ -#define JSON_MAX_DEPTH 2000 +#define JSON_MAX_DEPTH 1000 /************************************************************************** ** Utility routines for dealing with JsonString objects @@ -199057,6 +200019,129 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ assert( p->nUsednAlloc ); } +/* +** The zIn[0..N] string is a JSON5 string literal. Append to p a translation +** of the string literal that standard JSON and that omits all JSON5 +** features. +*/ +static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){ + u32 i; + jsonAppendChar(p, '"'); + zIn++; + N -= 2; + while( N>0 ){ + for(i=0; i0 ){ + jsonAppendRaw(p, zIn, i); + zIn += i; + N -= i; + if( N==0 ) break; + } + assert( zIn[0]=='\\' ); + switch( (u8)zIn[1] ){ + case '\'': + jsonAppendChar(p, '\''); + break; + case 'v': + jsonAppendRaw(p, "\\u0009", 6); + break; + case 'x': + jsonAppendRaw(p, "\\u00", 4); + jsonAppendRaw(p, &zIn[2], 2); + zIn += 2; + N -= 2; + break; + case '0': + jsonAppendRaw(p, "\\u0000", 6); + break; + case '\r': + if( zIn[2]=='\n' ){ + zIn++; + N--; + } + break; + case '\n': + break; + case 0xe2: + assert( N>=4 ); + assert( 0x80==(u8)zIn[2] ); + assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] ); + zIn += 2; + N -= 2; + break; + default: + jsonAppendRaw(p, zIn, 2); + break; + } + zIn += 2; + N -= 2; + } + jsonAppendChar(p, '"'); +} + +/* +** The zIn[0..N] string is a JSON5 integer literal. Append to p a translation +** of the string literal that standard JSON and that omits all JSON5 +** features. +*/ +static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){ + if( zIn[0]=='+' ){ + zIn++; + N--; + }else if( zIn[0]=='-' ){ + jsonAppendChar(p, '-'); + zIn++; + N--; + } + if( zIn[0]=='0' && (zIn[1]=='x' || zIn[1]=='X') ){ + sqlite3_int64 i = 0; + int rc = sqlite3DecOrHexToI64(zIn, &i); + if( rc<=1 ){ + jsonPrintf(100,p,"%lld",i); + }else{ + assert( rc==2 ); + jsonAppendRaw(p, "9.0e999", 7); + } + return; + } + jsonAppendRaw(p, zIn, N); +} + +/* +** The zIn[0..N] string is a JSON5 real literal. Append to p a translation +** of the string literal that standard JSON and that omits all JSON5 +** features. +*/ +static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){ + u32 i; + if( zIn[0]=='+' ){ + zIn++; + N--; + }else if( zIn[0]=='-' ){ + jsonAppendChar(p, '-'); + zIn++; + N--; + } + if( zIn[0]=='.' ){ + jsonAppendChar(p, '0'); + } + for(i=0; i0 ){ + jsonAppendRaw(p, zIn, N); + } +} + + + /* ** Append a function parameter value to the JSON string under ** construction. @@ -199070,8 +200155,11 @@ static void jsonAppendValue( jsonAppendRaw(p, "null", 4); break; } - case SQLITE_INTEGER: case SQLITE_FLOAT: { + jsonPrintf(100, p, "%!0.15g", sqlite3_value_double(pValue)); + break; + } + case SQLITE_INTEGER: { const char *z = (const char*)sqlite3_value_text(pValue); u32 n = (u32)sqlite3_value_bytes(pValue); jsonAppendRaw(p, z, n); @@ -199184,17 +200272,38 @@ static void jsonRenderNode( break; } case JSON_STRING: { + assert( pNode->eU==1 ); if( pNode->jnFlags & JNODE_RAW ){ - assert( pNode->eU==1 ); - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - break; + if( pNode->jnFlags & JNODE_LABEL ){ + jsonAppendChar(pOut, '"'); + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + jsonAppendChar(pOut, '"'); + }else{ + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + } + }else if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n); + }else{ + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); } - /* no break */ deliberate_fall_through + break; + } + case JSON_REAL: { + assert( pNode->eU==1 ); + if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n); + }else{ + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + } + break; } - case JSON_REAL: case JSON_INT: { assert( pNode->eU==1 ); - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + if( pNode->jnFlags & JNODE_JSON5 ){ + jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n); + }else{ + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + } break; } case JSON_ARRAY: { @@ -199310,59 +200419,41 @@ static void jsonReturn( } case JSON_INT: { sqlite3_int64 i = 0; + int rc; + int bNeg = 0; const char *z; + + assert( pNode->eU==1 ); z = pNode->u.zJContent; - if( z[0]=='-' ){ z++; } - while( z[0]>='0' && z[0]<='9' ){ - unsigned v = *(z++) - '0'; - if( i>=LARGEST_INT64/10 ){ - if( i>LARGEST_INT64/10 ) goto int_as_real; - if( z[0]>='0' && z[0]<='9' ) goto int_as_real; - if( v==9 ) goto int_as_real; - if( v==8 ){ - if( pNode->u.zJContent[0]=='-' ){ - sqlite3_result_int64(pCtx, SMALLEST_INT64); - goto int_done; - }else{ - goto int_as_real; - } - } - } - i = i*10 + v; + if( z[0]=='-' ){ z++; bNeg = 1; } + else if( z[0]=='+' ){ z++; } + rc = sqlite3DecOrHexToI64(z, &i); + if( rc<=1 ){ + sqlite3_result_int64(pCtx, bNeg ? -i : i); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else{ + goto to_double; } - if( pNode->u.zJContent[0]=='-' ){ i = -i; } - sqlite3_result_int64(pCtx, i); - int_done: break; - int_as_real: ; /* no break */ deliberate_fall_through } case JSON_REAL: { double r; -#ifdef SQLITE_AMALGAMATION const char *z; assert( pNode->eU==1 ); + to_double: z = pNode->u.zJContent; sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); -#else - assert( pNode->eU==1 ); - r = strtod(pNode->u.zJContent, 0); -#endif sqlite3_result_double(pCtx, r); break; } case JSON_STRING: { -#if 0 /* Never happens because JNODE_RAW is only set by json_set(), - ** json_insert() and json_replace() and those routines do not - ** call jsonReturn() */ if( pNode->jnFlags & JNODE_RAW ){ assert( pNode->eU==1 ); sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, SQLITE_TRANSIENT); - }else -#endif - assert( (pNode->jnFlags & JNODE_RAW)==0 ); - if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ + }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ /* JSON formatted without any backslash-escapes */ assert( pNode->eU==1 ); sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, @@ -199374,18 +200465,17 @@ static void jsonReturn( const char *z; char *zOut; u32 j; + u32 nOut = n; assert( pNode->eU==1 ); z = pNode->u.zJContent; - zOut = sqlite3_malloc( n+1 ); + zOut = sqlite3_malloc( nOut+1 ); if( zOut==0 ){ sqlite3_result_error_nomem(pCtx); break; } for(i=1, j=0; iaNode[pParse->nNode]; - p->eType = (u8)eType; - p->jnFlags = 0; + p->eType = (u8)(eType & 0xff); + p->jnFlags = (u8)(eType >> 8); VVA( p->eU = zContent ? 1 : 0 ); p->n = n; p->u.zJContent = zContent; return pParse->nNode++; } +/* +** Return true if z[] begins with 2 (or more) hexadecimal digits +*/ +static int jsonIs2Hex(const char *z){ + return sqlite3Isxdigit(z[0]) && sqlite3Isxdigit(z[1]); +} + /* ** Return true if z[] begins with 4 (or more) hexadecimal digits */ static int jsonIs4Hex(const char *z){ - int i; - for(i=0; i<4; i++) if( !sqlite3Isxdigit(z[i]) ) return 0; - return 1; + return jsonIs2Hex(z) && jsonIs2Hex(&z[2]); +} + +/* +** Return the number of bytes of JSON5 whitespace at the beginning of +** the input string z[]. +** +** JSON5 whitespace consists of any of the following characters: +** +** Unicode UTF-8 Name +** U+0009 09 horizontal tab +** U+000a 0a line feed +** U+000b 0b vertical tab +** U+000c 0c form feed +** U+000d 0d carriage return +** U+0020 20 space +** U+00a0 c2 a0 non-breaking space +** U+1680 e1 9a 80 ogham space mark +** U+2000 e2 80 80 en quad +** U+2001 e2 80 81 em quad +** U+2002 e2 80 82 en space +** U+2003 e2 80 83 em space +** U+2004 e2 80 84 three-per-em space +** U+2005 e2 80 85 four-per-em space +** U+2006 e2 80 86 six-per-em space +** U+2007 e2 80 87 figure space +** U+2008 e2 80 88 punctuation space +** U+2009 e2 80 89 thin space +** U+200a e2 80 8a hair space +** U+2028 e2 80 a8 line separator +** U+2029 e2 80 a9 paragraph separator +** U+202f e2 80 af narrow no-break space (NNBSP) +** U+205f e2 81 9f medium mathematical space (MMSP) +** U+3000 e3 80 80 ideographical space +** U+FEFF ef bb bf byte order mark +** +** In addition, comments between '/', '*' and '*', '/' and +** from '/', '/' to end-of-line are also considered to be whitespace. +*/ +static int json5Whitespace(const char *zIn){ + int n = 0; + const u8 *z = (u8*)zIn; + while( 1 /*exit by "goto whitespace_done"*/ ){ + switch( z[n] ){ + case 0x09: + case 0x0a: + case 0x0b: + case 0x0c: + case 0x0d: + case 0x20: { + n++; + break; + } + case '/': { + if( z[n+1]=='*' && z[n+2]!=0 ){ + int j; + for(j=n+3; z[j]!='/' || z[j-1]!='*'; j++){ + if( z[j]==0 ) goto whitespace_done; + } + n = j+1; + break; + }else if( z[n+1]=='/' ){ + int j; + char c; + for(j=n+2; (c = z[j])!=0; j++){ + if( c=='\n' || c=='\r' ) break; + if( 0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2]) + ){ + j += 2; + break; + } + } + n = j; + if( z[n] ) n++; + break; + } + goto whitespace_done; + } + case 0xc2: { + if( z[n+1]==0xa0 ){ + n += 2; + break; + } + goto whitespace_done; + } + case 0xe1: { + if( z[n+1]==0x9a && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xe2: { + if( z[n+1]==0x80 ){ + u8 c = z[n+2]; + if( c<0x80 ) goto whitespace_done; + if( c<=0x8a || c==0xa8 || c==0xa9 || c==0xaf ){ + n += 3; + break; + } + }else if( z[n+1]==0x81 && z[n+2]==0x9f ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xe3: { + if( z[n+1]==0x80 && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xef: { + if( z[n+1]==0xbb && z[n+2]==0xbf ){ + n += 3; + break; + } + goto whitespace_done; + } + default: { + goto whitespace_done; + } + } + } + whitespace_done: + return n; } +/* +** Extra floating-point literals to allow in JSON. +*/ +static const struct NanInfName { + char c1; + char c2; + char n; + char eType; + char nRepl; + char *zMatch; + char *zRepl; +} aNanInfName[] = { + { 'i', 'I', 3, JSON_REAL, 7, "inf", "9.0e999" }, + { 'i', 'I', 8, JSON_REAL, 7, "infinity", "9.0e999" }, + { 'n', 'N', 3, JSON_NULL, 4, "NaN", "null" }, + { 'q', 'Q', 4, JSON_NULL, 4, "QNaN", "null" }, + { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" }, +}; + /* ** Parse a single JSON value which begins at pParse->zJson[i]. Return the ** index of the first character past the end of the value parsed. ** -** Return negative for a syntax error. Special cases: return -2 if the -** first non-whitespace character is '}' and return -3 if the first -** non-whitespace character is ']'. +** Special return values: +** +** 0 End if input +** -1 Syntax error +** -2 '}' seen +** -3 ']' seen +** -4 ',' seen +** -5 ':' seen */ static int jsonParseValue(JsonParse *pParse, u32 i){ char c; @@ -199532,151 +200796,430 @@ static int jsonParseValue(JsonParse *pParse, u32 i){ int x; JsonNode *pNode; const char *z = pParse->zJson; - while( fast_isspace(z[i]) ){ i++; } - if( (c = z[i])=='{' ){ +json_parse_restart: + switch( (u8)z[i] ){ + case '{': { /* Parse object */ iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); if( iThis<0 ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } for(j=i+1;;j++){ - while( fast_isspace(z[j]) ){ j++; } - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + u32 nNode = pParse->nNode; x = jsonParseValue(pParse, j); - if( x<0 ){ - pParse->iDepth--; - if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; - return -1; + if( x<=0 ){ + if( x==(-2) ){ + j = pParse->iErr; + if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; + break; + } + j += json5Whitespace(&z[j]); + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && z[j+1]=='u' && jsonIs4Hex(&z[j+2])) + ){ + int k = j+1; + while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) + || (z[k]=='\\' && z[k+1]=='u' && jsonIs4Hex(&z[k+2])) + ){ + k++; + } + jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), k-j, &z[j]); + pParse->hasNonstd = 1; + x = k; + }else{ + if( x!=-1 ) pParse->iErr = j; + return -1; + } } if( pParse->oom ) return -1; - pNode = &pParse->aNode[pParse->nNode-1]; - if( pNode->eType!=JSON_STRING ) return -1; + pNode = &pParse->aNode[nNode]; + if( pNode->eType!=JSON_STRING ){ + pParse->iErr = j; + return -1; + } pNode->jnFlags |= JNODE_LABEL; j = x; - while( fast_isspace(z[j]) ){ j++; } - if( z[j]!=':' ) return -1; - j++; + if( z[j]==':' ){ + j++; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==':' ){ + j++; + goto parse_object_value; + } + } + x = jsonParseValue(pParse, j); + if( x!=(-5) ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = pParse->iErr+1; + } + parse_object_value: x = jsonParseValue(pParse, j); - pParse->iDepth--; - if( x<0 ) return -1; + if( x<=0 ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } j = x; - while( fast_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!='}' ) return -1; - break; + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + } + } + x = jsonParseValue(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-2) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + pParse->iDepth--; return j+1; - }else if( c=='[' ){ + } + case '[': { /* Parse array */ iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); if( iThis<0 ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u)); for(j=i+1;;j++){ - while( fast_isspace(z[j]) ){ j++; } - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; x = jsonParseValue(pParse, j); - pParse->iDepth--; - if( x<0 ){ - if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; + if( x<=0 ){ + if( x==(-3) ){ + j = pParse->iErr; + if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1; + break; + } + if( x!=(-1) ) pParse->iErr = j; return -1; } j = x; - while( fast_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!=']' ) return -1; - break; + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + }else{ + if( fast_isspace(z[j]) ){ + do{ j++; }while( fast_isspace(z[j]) ); + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + } + } + x = jsonParseValue(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-3) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; } pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + pParse->iDepth--; return j+1; - }else if( c=='"' ){ + } + case '\'': { + u8 jnFlags; + char cDelim; + pParse->hasNonstd = 1; + jnFlags = JNODE_JSON5; + goto parse_string; + case '"': /* Parse string */ - u8 jnFlags = 0; + jnFlags = 0; + parse_string: + cDelim = z[i]; j = i+1; for(;;){ c = z[j]; if( (c & ~0x1f)==0 ){ /* Control characters are not allowed in strings */ + pParse->iErr = j; return -1; } if( c=='\\' ){ c = z[++j]; if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' || c=='n' || c=='r' || c=='t' - || (c=='u' && jsonIs4Hex(z+j+1)) ){ - jnFlags = JNODE_ESCAPE; + || (c=='u' && jsonIs4Hex(&z[j+1])) ){ + jnFlags |= JNODE_ESCAPE; + }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + || (0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) + || (c=='x' && jsonIs2Hex(&z[j+1])) ){ + jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); + pParse->hasNonstd = 1; + }else if( c=='\r' ){ + if( z[j+1]=='\n' ) j++; + jnFlags |= (JNODE_ESCAPE|JNODE_JSON5); + pParse->hasNonstd = 1; }else{ + pParse->iErr = j; return -1; } - }else if( c=='"' ){ + }else if( c==cDelim ){ break; } j++; } - jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); - if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]); return j+1; - }else if( c=='n' - && strncmp(z+i,"null",4)==0 - && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return i+4; - }else if( c=='t' - && strncmp(z+i,"true",4)==0 - && !sqlite3Isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - return i+4; - }else if( c=='f' - && strncmp(z+i,"false",5)==0 - && !sqlite3Isalnum(z[i+5]) ){ - jsonParseAddNode(pParse, JSON_FALSE, 0, 0); - return i+5; - }else if( c=='-' || (c>='0' && c<='9') ){ + } + case 't': { + if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + } + pParse->iErr = i; + return -1; + } + case 'f': { + if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + } + pParse->iErr = i; + return -1; + } + case '+': { + u8 seenDP, seenE, jnFlags; + pParse->hasNonstd = 1; + jnFlags = JNODE_JSON5; + goto parse_number; + case '.': + if( sqlite3Isdigit(z[i+1]) ){ + pParse->hasNonstd = 1; + jnFlags = JNODE_JSON5; + seenE = 0; + seenDP = JSON_REAL; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': /* Parse number */ - u8 seenDP = 0; - u8 seenE = 0; + jnFlags = 0; + parse_number: + seenDP = JSON_INT; + seenE = 0; assert( '-' < '0' ); + assert( '+' < '0' ); + assert( '.' < '0' ); + c = z[i]; + if( c<='0' ){ - j = c=='-' ? i+1 : i; - if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1; + if( c=='0' ){ + if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ + assert( seenDP==JSON_INT ); + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + for(j=i+3; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + }else if( sqlite3Isdigit(z[i+1]) ){ + pParse->iErr = i+1; + return -1; + } + }else{ + if( !sqlite3Isdigit(z[i+1]) ){ + /* JSON5 allows for "+Infinity" and "-Infinity" using exactly + ** that case. SQLite also allows these in any case and it allows + ** "+inf" and "-inf". */ + if( (z[i+1]=='I' || z[i+1]=='i') + && sqlite3StrNICmp(&z[i+1], "inf",3)==0 + ){ + pParse->hasNonstd = 1; + if( z[i]=='-' ){ + jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999"); + }else{ + jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999"); + } + return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); + } + if( z[i+1]=='.' ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + } + if( z[i+1]=='0' ){ + if( sqlite3Isdigit(z[i+2]) ){ + pParse->iErr = i+1; + return -1; + }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + for(j=i+4; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + } + } + } } - j = i+1; - for(;; j++){ + parse_number_2: + for(j=i+1;; j++){ c = z[j]; - if( c>='0' && c<='9' ) continue; + if( sqlite3Isdigit(c) ) continue; if( c=='.' ){ - if( z[j-1]=='-' ) return -1; - if( seenDP ) return -1; - seenDP = 1; + if( seenDP==JSON_REAL ){ + pParse->iErr = j; + return -1; + } + seenDP = JSON_REAL; continue; } if( c=='e' || c=='E' ){ - if( z[j-1]<'0' ) return -1; - if( seenE ) return -1; - seenDP = seenE = 1; + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + }else{ + pParse->iErr = j; + return -1; + } + } + if( seenE ){ + pParse->iErr = j; + return -1; + } + seenDP = JSON_REAL; + seenE = 1; c = z[j+1]; if( c=='+' || c=='-' ){ j++; c = z[j+1]; } - if( c<'0' || c>'9' ) return -1; + if( c<'0' || c>'9' ){ + pParse->iErr = j; + return -1; + } continue; } break; } - if( z[j-1]<'0' ) return -1; - jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, - j - i, &z[i]); + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + jnFlags |= JNODE_JSON5; + }else{ + pParse->iErr = j; + return -1; + } + } + parse_number_finish: + jsonParseAddNode(pParse, seenDP | (jnFlags<<8), j - i, &z[i]); return j; - }else if( c=='}' ){ + } + case '}': { + pParse->iErr = i; return -2; /* End of {...} */ - }else if( c==']' ){ + } + case ']': { + pParse->iErr = i; return -3; /* End of [...] */ - }else if( c==0 ){ + } + case ',': { + pParse->iErr = i; + return -4; /* List separator */ + } + case ':': { + pParse->iErr = i; + return -5; /* Object label/value separator */ + } + case 0: { return 0; /* End of file */ - }else{ + } + case 0x09: + case 0x0a: + case 0x0d: + case 0x20: { + do{ + i++; + }while( fast_isspace(z[i]) ); + goto json_parse_restart; + } + case 0x0b: + case 0x0c: + case '/': + case 0xc2: + case 0xe1: + case 0xe2: + case 0xe3: + case 0xef: { + j = json5Whitespace(&z[i]); + if( j>0 ){ + i += j; + pParse->hasNonstd = 1; + goto json_parse_restart; + } + pParse->iErr = i; + return -1; + } + case 'n': { + if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + } + /* fall-through into the default case that checks for NaN */ + } + default: { + u32 k; + int nn; + c = z[i]; + for(k=0; khasNonstd = 1; + return i + nn; + } + pParse->iErr = i; return -1; /* Syntax error */ } + } /* End switch(z[i]) */ } /* @@ -199700,7 +201243,14 @@ static int jsonParse( if( i>0 ){ assert( pParse->iDepth==0 ); while( fast_isspace(zJson[i]) ) i++; - if( zJson[i] ) i = -1; + if( zJson[i] ){ + i += json5Whitespace(&zJson[i]); + if( zJson[i] ){ + jsonParseReset(pParse); + return 1; + } + pParse->hasNonstd = 1; + } } if( i<=0 ){ if( pCtx!=0 ){ @@ -199771,6 +201321,15 @@ static int jsonParseFindParents(JsonParse *pParse){ ** is no longer valid, parse the JSON again and return the new parse, ** and also register the new parse so that it will be available for ** future sqlite3_get_auxdata() calls. +** +** If an error occurs and pErrCtx!=0 then report the error on pErrCtx +** and return NULL. +** +** If an error occurs and pErrCtx==0 then return the Parse object with +** JsonParse.nErr non-zero. If the caller invokes this routine with +** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller +** is responsible for invoking jsonParseFree() on the returned value. +** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0. */ static JsonParse *jsonParseCached( sqlite3_context *pCtx, @@ -199820,6 +201379,10 @@ static JsonParse *jsonParseCached( p->zJson = (char*)&p[1]; memcpy((char*)p->zJson, zJson, nJson+1); if( jsonParse(p, pErrCtx, p->zJson) ){ + if( pErrCtx==0 ){ + p->nErr = 1; + return p; + } sqlite3_free(p); return 0; } @@ -199834,7 +201397,7 @@ static JsonParse *jsonParseCached( ** Compare the OBJECT label at pNode against zKey,nKey. Return true on ** a match. */ -static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ +static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){ assert( pNode->eU==1 ); if( pNode->jnFlags & JNODE_RAW ){ if( pNode->n!=nKey ) return 0; @@ -199844,6 +201407,15 @@ static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; } } +static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){ + if( p1->jnFlags & JNODE_RAW ){ + return jsonLabelCompare(p2, p1->u.zJContent, p1->n); + }else if( p2->jnFlags & JNODE_RAW ){ + return jsonLabelCompare(p1, p2->u.zJContent, p2->n); + }else{ + return p1->n==p2->n && strncmp(p1->u.zJContent,p2->u.zJContent,p1->n)==0; + } +} /* forward declaration */ static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); @@ -200314,7 +201886,7 @@ static void jsonExtractFunc( zPath = (const char*)sqlite3_value_text(argv[1]); if( zPath==0 ) return; if( flags & JSON_ABPATH ){ - if( zPath[0]!='$' ){ + if( zPath[0]!='$' || (zPath[1]!='.' && zPath[1]!='[' && zPath[1]!=0) ){ /* The -> and ->> operators accept abbreviated PATH arguments. This ** is mostly for compatibility with PostgreSQL, but also for ** convenience. @@ -200405,12 +201977,10 @@ static JsonNode *jsonMergePatch( assert( pPatch[i].eU==1 ); nKey = pPatch[i].n; zKey = pPatch[i].u.zJContent; - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ assert( pTarget[j].eType==JSON_STRING ); assert( pTarget[j].jnFlags & JNODE_LABEL ); - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); - if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){ if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; if( pPatch[i+1].eType==JSON_NULL ){ pTarget[j+1].jnFlags |= JNODE_REMOVE; @@ -200697,8 +202267,8 @@ static void jsonTypeFunc( /* ** json_valid(JSON) ** -** Return 1 if JSON is a well-formed JSON string according to RFC-7159. -** Return 0 otherwise. +** Return 1 if JSON is a well-formed canonical JSON string according +** to RFC-7159. Return 0 otherwise. */ static void jsonValidFunc( sqlite3_context *ctx, @@ -200707,8 +202277,69 @@ static void jsonValidFunc( ){ JsonParse *p; /* The parse */ UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + p = jsonParseCached(ctx, argv, 0); + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(ctx); + sqlite3_free(p); + }else{ + sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0); + if( p->nErr ) jsonParseFree(p); + } +} + +/* +** json_error_position(JSON) +** +** If the argument is not an interpretable JSON string, then return the 1-based +** character position at which the parser first recognized that the input +** was in error. The left-most character is 1. If the string is valid +** JSON, then return 0. +** +** Note that json_valid() is only true for strictly conforming canonical JSON. +** But this routine returns zero if the input contains extension. Thus: +** +** (1) If the input X is strictly conforming canonical JSON: +** +** json_valid(X) returns true +** json_error_position(X) returns 0 +** +** (2) If the input X is JSON but it includes extension (such as JSON5) that +** are not part of RFC-8259: +** +** json_valid(X) returns false +** json_error_position(X) return 0 +** +** (3) If the input X cannot be interpreted as JSON even taking extensions +** into account: +** +** json_valid(X) return false +** json_error_position(X) returns 1 or more +*/ +static void jsonErrorFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; p = jsonParseCached(ctx, argv, 0); - sqlite3_result_int(ctx, p!=0); + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(ctx); + sqlite3_free(p); + }else if( p->nErr==0 ){ + sqlite3_result_int(ctx, 0); + }else{ + int n = 1; + u32 i; + const char *z = p->zJson; + for(i=0; iiErr && ALWAYS(z[i]); i++){ + if( (z[i]&0xc0)!=0x80 ) n++; + } + sqlite3_result_int(ctx, n); + jsonParseFree(p); + } } @@ -201052,14 +202683,16 @@ static void jsonAppendObjectPathElement( assert( pNode->eU==1 ); z = pNode->u.zJContent; nn = pNode->n; - assert( nn>=2 ); - assert( z[0]=='"' ); - assert( z[nn-1]=='"' ); - if( nn>2 && sqlite3Isalpha(z[1]) ){ - for(jj=2; jjjnFlags & JNODE_RAW)==0 ){ + assert( nn>=2 ); + assert( z[0]=='"' || z[0]=='\'' ); + assert( z[nn-1]=='"' || z[0]=='\'' ); + if( nn>2 && sqlite3Isalpha(z[1]) ){ + for(jj=2; jj, 2, JSON_JSON, jsonExtractFunc), JFUNCTION(->>, 2, JSON_SQL, jsonExtractFunc), @@ -201943,16 +203577,17 @@ struct RtreeMatchArg { ** at run-time. */ #ifndef SQLITE_BYTEORDER -#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) -# define SQLITE_BYTEORDER 1234 -#elif defined(sparc) || defined(__ppc__) -# define SQLITE_BYTEORDER 4321 -#else -# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ -#endif +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) || \ + defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif #endif @@ -212497,6 +214132,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff); } +/* +** This value is copied from the definition of ZIPVFS_CTRL_FILE_POINTER +** in zipvfs.h. +*/ +#define RBU_ZIPVFS_CTRL_FILE_POINTER 230439 /* ** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if @@ -212505,9 +214145,20 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ static int rbuLockDatabase(sqlite3 *db){ int rc = SQLITE_OK; sqlite3_file *fd = 0; - sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); - if( fd->pMethods ){ + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + if( fd ){ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); + if( rc==SQLITE_OK ){ + rc = fd->pMethods->xUnlock(fd, SQLITE_LOCK_NONE); + } + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + }else{ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + } + + if( rc==SQLITE_OK && fd->pMethods ){ rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); if( rc==SQLITE_OK ){ rc = fd->pMethods->xLock(fd, SQLITE_LOCK_EXCLUSIVE); @@ -215744,6 +217395,7 @@ static int dbpageConnect( (void)pzErr; sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); rc = sqlite3_declare_vtab(db, "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); if( rc==SQLITE_OK ){ @@ -215827,7 +217479,6 @@ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ pIdxInfo->orderByConsumed = 1; } - sqlite3VtabUsesAllSchemas(pIdxInfo); return SQLITE_OK; } @@ -216128,6 +217779,8 @@ typedef struct SessionInput SessionInput; # endif #endif +#define SESSIONS_ROWID "_rowid_" + static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE; typedef struct SessionHook SessionHook; @@ -216149,6 +217802,7 @@ struct sqlite3_session { int bEnable; /* True if currently recording */ int bIndirect; /* True if all changes are indirect */ int bAutoAttach; /* True to auto-attach tables */ + int bImplicitPK; /* True to handle tables with implicit PK */ int rc; /* Non-zero if an error has occurred */ void *pFilterCtx; /* First argument to pass to xTableFilter */ int (*xTableFilter)(void *pCtx, const char *zTab); @@ -216225,6 +217879,7 @@ struct SessionTable { char *zName; /* Local name of table */ int nCol; /* Number of columns in table zName */ int bStat1; /* True if this is sqlite_stat1 */ + int bRowid; /* True if this table uses rowid for PK */ const char **azCol; /* Column names */ u8 *abPK; /* Array of primary key flags */ int nEntry; /* Total number of entries in hash table */ @@ -216617,6 +218272,7 @@ static unsigned int sessionHashAppendType(unsigned int h, int eType){ */ static int sessionPreupdateHash( sqlite3_session *pSession, /* Session object that owns pTab */ + i64 iRowid, SessionTable *pTab, /* Session table handle */ int bNew, /* True to hash the new.* PK */ int *piHash, /* OUT: Hash value */ @@ -216625,48 +218281,53 @@ static int sessionPreupdateHash( unsigned int h = 0; /* Hash value to return */ int i; /* Used to iterate through columns */ - assert( *pbNullPK==0 ); - assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); - for(i=0; inCol; i++){ - if( pTab->abPK[i] ){ - int rc; - int eType; - sqlite3_value *pVal; - - if( bNew ){ - rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); - }else{ - rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); - } - if( rc!=SQLITE_OK ) return rc; + if( pTab->bRowid ){ + assert( pTab->nCol-1==pSession->hook.xCount(pSession->hook.pCtx) ); + h = sessionHashAppendI64(h, iRowid); + }else{ + assert( *pbNullPK==0 ); + assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); + for(i=0; inCol; i++){ + if( pTab->abPK[i] ){ + int rc; + int eType; + sqlite3_value *pVal; - eType = sqlite3_value_type(pVal); - h = sessionHashAppendType(h, eType); - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - i64 iVal; - if( eType==SQLITE_INTEGER ){ - iVal = sqlite3_value_int64(pVal); + if( bNew ){ + rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); }else{ - double rVal = sqlite3_value_double(pVal); - assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); - memcpy(&iVal, &rVal, 8); + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); } - h = sessionHashAppendI64(h, iVal); - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ - const u8 *z; - int n; - if( eType==SQLITE_TEXT ){ - z = (const u8 *)sqlite3_value_text(pVal); + if( rc!=SQLITE_OK ) return rc; + + eType = sqlite3_value_type(pVal); + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_value_int64(pVal); + }else{ + double rVal = sqlite3_value_double(pVal); + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&iVal, &rVal, 8); + } + h = sessionHashAppendI64(h, iVal); + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + const u8 *z; + int n; + if( eType==SQLITE_TEXT ){ + z = (const u8 *)sqlite3_value_text(pVal); + }else{ + z = (const u8 *)sqlite3_value_blob(pVal); + } + n = sqlite3_value_bytes(pVal); + if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + h = sessionHashAppendBlob(h, n, z); }else{ - z = (const u8 *)sqlite3_value_blob(pVal); + assert( eType==SQLITE_NULL ); + assert( pTab->bStat1==0 || i!=1 ); + *pbNullPK = 1; } - n = sqlite3_value_bytes(pVal); - if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; - h = sessionHashAppendBlob(h, n, z); - }else{ - assert( eType==SQLITE_NULL ); - assert( pTab->bStat1==0 || i!=1 ); - *pbNullPK = 1; } } } @@ -216949,6 +218610,7 @@ static int sessionMergeUpdate( */ static int sessionPreupdateEqual( sqlite3_session *pSession, /* Session object that owns SessionTable */ + i64 iRowid, /* Rowid value if pTab->bRowid */ SessionTable *pTab, /* Table associated with change */ SessionChange *pChange, /* Change to compare to */ int op /* Current pre-update operation */ @@ -216956,6 +218618,11 @@ static int sessionPreupdateEqual( int iCol; /* Used to iterate through columns */ u8 *a = pChange->aRecord; /* Cursor used to scan change record */ + if( pTab->bRowid ){ + if( a[0]!=SQLITE_INTEGER ) return 0; + return sessionGetI64(&a[1])==iRowid; + } + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); for(iCol=0; iColnCol; iCol++){ if( !pTab->abPK[iCol] ){ @@ -217100,7 +218767,8 @@ static int sessionTableInfo( int *pnCol, /* OUT: number of columns */ const char **pzTab, /* OUT: Copy of zThis */ const char ***pazCol, /* OUT: Array of column names for table */ - u8 **pabPK /* OUT: Array of booleans - true for PK col */ + u8 **pabPK, /* OUT: Array of booleans - true for PK col */ + int *pbRowid /* OUT: True if only PK is a rowid */ ){ char *zPragma; sqlite3_stmt *pStmt; @@ -217112,6 +218780,7 @@ static int sessionTableInfo( u8 *pAlloc = 0; char **azCol = 0; u8 *abPK = 0; + int bRowid = 0; /* Set to true to use rowid as PK */ assert( pazCol && pabPK ); @@ -217156,10 +218825,15 @@ static int sessionTableInfo( } nByte = nThis + 1; + bRowid = (pbRowid!=0); while( SQLITE_ROW==sqlite3_step(pStmt) ){ nByte += sqlite3_column_bytes(pStmt, 1); nDbCol++; + if( sqlite3_column_int(pStmt, 5) ) bRowid = 0; } + if( nDbCol==0 ) bRowid = 0; + nDbCol += bRowid; + nByte += strlen(SESSIONS_ROWID); rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ){ @@ -217181,6 +218855,14 @@ static int sessionTableInfo( } i = 0; + if( bRowid ){ + size_t nName = strlen(SESSIONS_ROWID); + memcpy(pAlloc, SESSIONS_ROWID, nName+1); + azCol[i] = (char*)pAlloc; + pAlloc += nName+1; + abPK[i] = 1; + i++; + } while( SQLITE_ROW==sqlite3_step(pStmt) ){ int nName = sqlite3_column_bytes(pStmt, 1); const unsigned char *zName = sqlite3_column_text(pStmt, 1); @@ -217192,7 +218874,6 @@ static int sessionTableInfo( i++; } rc = sqlite3_reset(pStmt); - } /* If successful, populate the output variables. Otherwise, zero them and @@ -217209,6 +218890,7 @@ static int sessionTableInfo( if( pzTab ) *pzTab = 0; sessionFree(pSession, azCol); } + if( pbRowid ) *pbRowid = bRowid; sqlite3_finalize(pStmt); return rc; } @@ -217230,7 +218912,8 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ u8 *abPK; assert( pTab->azCol==0 || pTab->abPK==0 ); pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, - pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK + pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK, + (pSession->bImplicitPK ? &pTab->bRowid : 0) ); if( pSession->rc==SQLITE_OK ){ int i; @@ -217302,6 +218985,7 @@ static int sessionUpdateMaxSize( ){ i64 nNew = 2; if( pC->op==SQLITE_INSERT ){ + if( pTab->bRowid ) nNew += 9; if( op!=SQLITE_DELETE ){ int ii; for(ii=0; iinCol; ii++){ @@ -217318,12 +219002,16 @@ static int sessionUpdateMaxSize( }else{ int ii; u8 *pCsr = pC->aRecord; - for(ii=0; iinCol; ii++){ + if( pTab->bRowid ){ + nNew += 9 + 1; + pCsr += 9; + } + for(ii=pTab->bRowid; iinCol; ii++){ int bChanged = 1; int nOld = 0; int eType; sqlite3_value *p = 0; - pSession->hook.xNew(pSession->hook.pCtx, ii, &p); + pSession->hook.xNew(pSession->hook.pCtx, ii-pTab->bRowid, &p); if( p==0 ){ return SQLITE_NOMEM; } @@ -217402,6 +219090,7 @@ static int sessionUpdateMaxSize( */ static void sessionPreupdateOneChange( int op, /* One of SQLITE_UPDATE, INSERT, DELETE */ + i64 iRowid, sqlite3_session *pSession, /* Session object pTab is attached to */ SessionTable *pTab /* Table that change applies to */ ){ @@ -217417,7 +219106,7 @@ static void sessionPreupdateOneChange( /* Check the number of columns in this xPreUpdate call matches the ** number of columns in the table. */ - if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){ + if( (pTab->nCol-pTab->bRowid)!=pSession->hook.xCount(pSession->hook.pCtx) ){ pSession->rc = SQLITE_SCHEMA; return; } @@ -217450,14 +219139,16 @@ static void sessionPreupdateOneChange( /* Calculate the hash-key for this change. If the primary key of the row ** includes a NULL value, exit early. Such changes are ignored by the ** session module. */ - rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull); + rc = sessionPreupdateHash( + pSession, iRowid, pTab, op==SQLITE_INSERT, &iHash, &bNull + ); if( rc!=SQLITE_OK ) goto error_out; if( bNull==0 ){ /* Search the hash table for an existing record for this row. */ SessionChange *pC; for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){ - if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break; + if( sessionPreupdateEqual(pSession, iRowid, pTab, pC, op) ) break; } if( pC==0 ){ @@ -217472,7 +219163,7 @@ static void sessionPreupdateOneChange( /* Figure out how large an allocation is required */ nByte = sizeof(SessionChange); - for(i=0; inCol; i++){ + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -217487,6 +219178,9 @@ static void sessionPreupdateOneChange( rc = sessionSerializeValue(0, p, &nByte); if( rc!=SQLITE_OK ) goto error_out; } + if( pTab->bRowid ){ + nByte += 9; /* Size of rowid field - an integer */ + } /* Allocate the change object */ pC = (SessionChange *)sessionMalloc64(pSession, nByte); @@ -217503,7 +219197,12 @@ static void sessionPreupdateOneChange( ** required values and encodings have already been cached in memory. ** It is not possible for an OOM to occur in this block. */ nByte = 0; - for(i=0; inCol; i++){ + if( pTab->bRowid ){ + pC->aRecord[0] = SQLITE_INTEGER; + sessionPutI64(&pC->aRecord[1], iRowid); + nByte = 9; + } + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ pSession->hook.xOld(pSession->hook.pCtx, i, &p); @@ -217618,9 +219317,10 @@ static void xPreUpdate( pSession->rc = sessionFindTable(pSession, zName, &pTab); if( pTab ){ assert( pSession->rc==SQLITE_OK ); - sessionPreupdateOneChange(op, pSession, pTab); + assert( op==SQLITE_UPDATE || iKey1==iKey2 ); + sessionPreupdateOneChange(op, iKey1, pSession, pTab); if( op==SQLITE_UPDATE ){ - sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab); + sessionPreupdateOneChange(SQLITE_INSERT, iKey2, pSession, pTab); } } } @@ -217659,6 +219359,7 @@ static void sessionPreupdateHooks( typedef struct SessionDiffCtx SessionDiffCtx; struct SessionDiffCtx { sqlite3_stmt *pStmt; + int bRowid; int nOldOff; }; @@ -217667,17 +219368,17 @@ struct SessionDiffCtx { */ static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff+p->bRowid); return SQLITE_OK; } static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->bRowid); return SQLITE_OK; } static int sessionDiffCount(void *pCtx){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt); + return (p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt)) - p->bRowid; } static int sessionDiffDepth(void *pCtx){ (void)pCtx; @@ -217756,14 +219457,16 @@ static char *sessionExprCompareOther( static char *sessionSelectFindNew( const char *zDb1, /* Pick rows in this db only */ const char *zDb2, /* But not in this one */ + int bRowid, const char *zTbl, /* Table name */ const char *zExpr ){ + const char *zSel = (bRowid ? SESSIONS_ROWID ", *" : "*"); char *zRet = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\" WHERE NOT EXISTS (" + "SELECT %s FROM \"%w\".\"%w\" WHERE NOT EXISTS (" " SELECT 1 FROM \"%w\".\"%w\" WHERE %s" ")", - zDb1, zTbl, zDb2, zTbl, zExpr + zSel, zDb1, zTbl, zDb2, zTbl, zExpr ); return zRet; } @@ -217777,7 +219480,9 @@ static int sessionDiffFindNew( char *zExpr ){ int rc = SQLITE_OK; - char *zStmt = sessionSelectFindNew(zDb1, zDb2, pTab->zName,zExpr); + char *zStmt = sessionSelectFindNew( + zDb1, zDb2, pTab->bRowid, pTab->zName, zExpr + ); if( zStmt==0 ){ rc = SQLITE_NOMEM; @@ -217788,8 +219493,10 @@ static int sessionDiffFindNew( SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = 0; + pDiffCtx->bRowid = pTab->bRowid; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(op, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(op, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } @@ -217799,6 +219506,27 @@ static int sessionDiffFindNew( return rc; } +/* +** Return a comma-separated list of the fully-qualified (with both database +** and table name) column names from table pTab. e.g. +** +** "main"."t1"."a", "main"."t1"."b", "main"."t1"."c" +*/ +static char *sessionAllCols( + const char *zDb, + SessionTable *pTab +){ + int ii; + char *zRet = 0; + for(ii=0; iinCol; ii++){ + zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"", + zRet, (zRet ? ", " : ""), zDb, pTab->zName, pTab->azCol[ii] + ); + if( !zRet ) break; + } + return zRet; +} + static int sessionDiffFindModified( sqlite3_session *pSession, SessionTable *pTab, @@ -217813,11 +219541,13 @@ static int sessionDiffFindModified( if( zExpr2==0 ){ rc = SQLITE_NOMEM; }else{ + char *z1 = sessionAllCols(pSession->zDb, pTab); + char *z2 = sessionAllCols(zFrom, pTab); char *zStmt = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", - pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 + "SELECT %s,%s FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", + z1, z2, pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 ); - if( zStmt==0 ){ + if( zStmt==0 || z1==0 || z2==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_stmt *pStmt; @@ -217828,12 +219558,15 @@ static int sessionDiffFindModified( pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = pTab->nCol; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(SQLITE_UPDATE, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } - sqlite3_free(zStmt); } + sqlite3_free(zStmt); + sqlite3_free(z1); + sqlite3_free(z2); } return rc; @@ -217872,9 +219605,12 @@ SQLITE_API int sqlite3session_diff( int bHasPk = 0; int bMismatch = 0; int nCol; /* Columns in zFrom.zTbl */ + int bRowid = 0; u8 *abPK; const char **azCol = 0; - rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK); + rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK, + pSession->bImplicitPK ? &bRowid : 0 + ); if( rc==SQLITE_OK ){ if( pTo->nCol!=nCol ){ bMismatch = 1; @@ -218216,9 +219952,10 @@ static void sessionAppendStr( int *pRc ){ int nStr = sqlite3Strlen30(zStr); - if( 0==sessionBufferGrow(p, nStr, pRc) ){ + if( 0==sessionBufferGrow(p, nStr+1, pRc) ){ memcpy(&p->aBuf[p->nBuf], zStr, nStr); p->nBuf += nStr; + p->aBuf[p->nBuf] = 0x00; } } @@ -218240,6 +219977,27 @@ static void sessionAppendInteger( sessionAppendStr(p, aBuf, pRc); } +static void sessionAppendPrintf( + SessionBuffer *p, /* Buffer to append to */ + int *pRc, + const char *zFmt, + ... +){ + if( *pRc==SQLITE_OK ){ + char *zApp = 0; + va_list ap; + va_start(ap, zFmt); + zApp = sqlite3_vmprintf(zFmt, ap); + if( zApp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sessionAppendStr(p, zApp, pRc); + } + va_end(ap); + sqlite3_free(zApp); + } +} + /* ** This function is a no-op if *pRc is other than SQLITE_OK when it is ** called. Otherwise, append the string zStr enclosed in quotes (") and @@ -218254,7 +220012,7 @@ static void sessionAppendIdent( const char *zStr, /* String to quote, escape and append */ int *pRc /* IN/OUT: Error code */ ){ - int nStr = sqlite3Strlen30(zStr)*2 + 2 + 1; + int nStr = sqlite3Strlen30(zStr)*2 + 2 + 2; if( 0==sessionBufferGrow(p, nStr, pRc) ){ char *zOut = (char *)&p->aBuf[p->nBuf]; const char *zIn = zStr; @@ -218265,6 +220023,7 @@ static void sessionAppendIdent( } *zOut++ = '"'; p->nBuf = (int)((u8 *)zOut - p->aBuf); + p->aBuf[p->nBuf] = 0x00; } } @@ -218400,7 +220159,7 @@ static int sessionAppendUpdate( /* If at least one field has been modified, this is not a no-op. */ if( bChanged ) bNoop = 0; - /* Add a field to the old.* record. This is omitted if this modules is + /* Add a field to the old.* record. This is omitted if this module is ** currently generating a patchset. */ if( bPatchset==0 ){ if( bChanged || abPK[i] ){ @@ -218489,12 +220248,20 @@ static int sessionAppendDelete( ** Formulate and prepare a SELECT statement to retrieve a row from table ** zTab in database zDb based on its primary key. i.e. ** -** SELECT * FROM zDb.zTab WHERE pk1 = ? AND pk2 = ? AND ... +** SELECT *, FROM zDb.zTab WHERE (pk1, pk2,...) IS (?1, ?2,...) +** +** where is: +** +** 1 AND (?A OR ?1 IS ) AND ... +** +** for each non-pk . */ static int sessionSelectStmt( sqlite3 *db, /* Database handle */ + int bIgnoreNoop, const char *zDb, /* Database name */ const char *zTab, /* Table name */ + int bRowid, int nCol, /* Number of columns in table */ const char **azCol, /* Names of table columns */ u8 *abPK, /* PRIMARY KEY array */ @@ -218502,8 +220269,50 @@ static int sessionSelectStmt( ){ int rc = SQLITE_OK; char *zSql = 0; + const char *zSep = ""; + const char *zCols = bRowid ? SESSIONS_ROWID ", *" : "*"; int nSql = -1; + int i; + + SessionBuffer nooptest = {0, 0, 0}; + SessionBuffer pkfield = {0, 0, 0}; + SessionBuffer pkvar = {0, 0, 0}; + + sessionAppendStr(&nooptest, ", 1", &rc); + if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){ + sessionAppendStr(&nooptest, " AND (?6 OR ?3 IS stat)", &rc); + sessionAppendStr(&pkfield, "tbl, idx", &rc); + sessionAppendStr(&pkvar, + "?1, (CASE WHEN ?2=X'' THEN NULL ELSE ?2 END)", &rc + ); + zCols = "tbl, ?2, stat"; + }else{ + for(i=0; izDb, zName, &nCol, 0,&azCol,&abPK); - if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){ + rc = sessionTableInfo( + 0, db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK, + (pSession->bImplicitPK ? &bRowid : 0) + ); + if( rc==SQLITE_OK && ( + pTab->nCol!=nCol + || pTab->bRowid!=bRowid + || memcmp(abPK, pTab->abPK, nCol) + )){ rc = SQLITE_SCHEMA; } @@ -218696,7 +220516,8 @@ static int sessionGenerateChangeset( /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ rc = sessionSelectStmt( - db, pSession->zDb, zName, nCol, azCol, abPK, &pSel); + db, 0, pSession->zDb, zName, bRowid, nCol, azCol, abPK, &pSel + ); } nNoop = buf.nBuf; @@ -218779,7 +220600,7 @@ SQLITE_API int sqlite3session_changeset( int rc; if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE; - rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset,ppChangeset); + rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); assert( rc || pnChangeset==0 || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize ); @@ -218897,6 +220718,19 @@ SQLITE_API int sqlite3session_object_config(sqlite3_session *pSession, int op, v break; } + case SQLITE_SESSION_OBJCONFIG_ROWID: { + int iArg = *(int*)pArg; + if( iArg>=0 ){ + if( pSession->pTable ){ + rc = SQLITE_MISUSE; + }else{ + pSession->bImplicitPK = (iArg!=0); + } + } + *(int*)pArg = pSession->bImplicitPK; + break; + } + default: rc = SQLITE_MISUSE; } @@ -219885,6 +221719,8 @@ struct SessionApplyCtx { SessionBuffer rebase; /* Rebase information (if any) here */ u8 bRebaseStarted; /* If table header is already in rebase */ u8 bRebase; /* True to collect rebase information */ + u8 bIgnoreNoop; /* True to ignore no-op conflicts */ + int bRowid; }; /* Number of prepared UPDATE statements to cache. */ @@ -220135,8 +221971,10 @@ static int sessionSelectRow( const char *zTab, /* Table name */ SessionApplyCtx *p /* Session changeset-apply context */ ){ - return sessionSelectStmt( - db, "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect); + /* TODO */ + return sessionSelectStmt(db, p->bIgnoreNoop, + "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect + ); } /* @@ -220295,20 +222133,33 @@ static int sessionBindRow( */ static int sessionSeekToRow( sqlite3_changeset_iter *pIter, /* Changeset iterator */ - u8 *abPK, /* Primary key flags array */ - sqlite3_stmt *pSelect /* SELECT statement from sessionSelectRow() */ + SessionApplyCtx *p ){ + sqlite3_stmt *pSelect = p->pSelect; int rc; /* Return code */ int nCol; /* Number of columns in table */ int op; /* Changset operation (SQLITE_UPDATE etc.) */ const char *zDummy; /* Unused */ + sqlite3_clear_bindings(pSelect); sqlite3changeset_op(pIter, &zDummy, &nCol, &op, 0); rc = sessionBindRow(pIter, op==SQLITE_INSERT ? sqlite3changeset_new : sqlite3changeset_old, - nCol, abPK, pSelect + nCol, p->abPK, pSelect ); + if( op!=SQLITE_DELETE && p->bIgnoreNoop ){ + int ii; + for(ii=0; rc==SQLITE_OK && iiabPK[ii]==0 ){ + sqlite3_value *pVal = 0; + sqlite3changeset_new(pIter, ii, &pVal); + sqlite3_bind_int(pSelect, ii+1+nCol, (pVal==0)); + if( pVal ) rc = sessionBindValue(pSelect, ii+1, pVal); + } + } + } + if( rc==SQLITE_OK ){ rc = sqlite3_step(pSelect); if( rc!=SQLITE_ROW ) rc = sqlite3_reset(pSelect); @@ -220423,16 +222274,22 @@ static int sessionConflictHandler( /* Bind the new.* PRIMARY KEY values to the SELECT statement. */ if( pbReplace ){ - rc = sessionSeekToRow(pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); }else{ rc = SQLITE_OK; } if( rc==SQLITE_ROW ){ /* There exists another row with the new.* primary key. */ - pIter->pConflict = p->pSelect; - res = xConflict(pCtx, eType, pIter); - pIter->pConflict = 0; + if( p->bIgnoreNoop + && sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) + ){ + res = SQLITE_CHANGESET_OMIT; + }else{ + pIter->pConflict = p->pSelect; + res = xConflict(pCtx, eType, pIter); + pIter->pConflict = 0; + } rc = sqlite3_reset(p->pSelect); }else if( rc==SQLITE_OK ){ if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){ @@ -220540,7 +222397,7 @@ static int sessionApplyOneOp( sqlite3_step(p->pDelete); rc = sqlite3_reset(p->pDelete); - if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 && p->bIgnoreNoop==0 ){ rc = sessionConflictHandler( SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry ); @@ -220597,7 +222454,7 @@ static int sessionApplyOneOp( /* Check if there is a conflicting row. For sqlite_stat1, this needs ** to be done using a SELECT, as there is no PRIMARY KEY in the ** database schema to throw an exception if a duplicate is inserted. */ - rc = sessionSeekToRow(pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); if( rc==SQLITE_ROW ){ rc = SQLITE_CONSTRAINT; sqlite3_reset(p->pSelect); @@ -220774,6 +222631,7 @@ static int sessionChangesetApply( memset(&sApply, 0, sizeof(sApply)); sApply.bRebase = (ppRebase && pnRebase); sApply.bInvertConstraints = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + sApply.bIgnoreNoop = !!(flags & SQLITE_CHANGESETAPPLY_IGNORENOOP); sqlite3_mutex_enter(sqlite3_db_mutex(db)); if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){ rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); @@ -220811,6 +222669,7 @@ static int sessionChangesetApply( sApply.bStat1 = 0; sApply.bDeferConstraints = 1; sApply.bRebaseStarted = 0; + sApply.bRowid = 0; memset(&sApply.constraints, 0, sizeof(SessionBuffer)); /* If an xFilter() callback was specified, invoke it now. If the @@ -220830,8 +222689,8 @@ static int sessionChangesetApply( int i; sqlite3changeset_pk(pIter, &abPK, 0); - rc = sessionTableInfo(0, - db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK + rc = sessionTableInfo(0, db, "main", zNew, + &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK, &sApply.bRowid ); if( rc!=SQLITE_OK ) break; for(i=0; iiPos++; - if( p->iRangeEnd>0 ){ + if( p->iRangeEnd>=0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; } @@ -225042,7 +226907,7 @@ static int fts5HighlightCb( } if( iPos==p->iter.iEnd ){ - if( p->iRangeEnd && p->iter.iStartiRangeStart ){ + if( p->iRangeEnd>=0 && p->iter.iStartiRangeStart ){ fts5HighlightAppend(&rc, p, p->zOpen, -1); } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); @@ -225053,7 +226918,7 @@ static int fts5HighlightCb( } } - if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ + if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; if( iPos>=p->iter.iStart && iPositer.iEnd ){ @@ -225088,6 +226953,7 @@ static void fts5HighlightFunction( memset(&ctx, 0, sizeof(HighlightContext)); ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + ctx.iRangeEnd = -1; rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); if( ctx.zIn ){ @@ -225273,6 +227139,7 @@ static void fts5SnippetFunction( iCol = sqlite3_value_int(apVal[0]); ctx.zOpen = fts5ValueToText(apVal[1]); ctx.zClose = fts5ValueToText(apVal[2]); + ctx.iRangeEnd = -1; zEllips = fts5ValueToText(apVal[3]); nToken = sqlite3_value_int(apVal[4]); @@ -226541,6 +228408,7 @@ static int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; } + assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK ); for(i=3; rc==SQLITE_OK && ibSecureDelete = (bVal ? 1 : 0); + } }else{ *pbBadkey = 1; } @@ -226938,15 +228818,20 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ rc = sqlite3_finalize(p); } - if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ + if( rc==SQLITE_OK + && iVersion!=FTS5_CURRENT_VERSION + && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE + ){ rc = SQLITE_ERROR; if( pConfig->pzErrmsg ){ assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf( - "invalid fts5 file format (found %d, expected %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION + *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE ); } + }else{ + pConfig->iVersion = iVersion; } if( rc==SQLITE_OK ){ @@ -226974,6 +228859,10 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ /* #include "fts5Int.h" */ /* #include "fts5parse.h" */ +#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH +# define SQLITE_FTS5_MAX_EXPR_DEPTH 256 +#endif + /* ** All token types in the generated fts5parse.h file are greater than 0. */ @@ -227014,11 +228903,17 @@ struct Fts5Expr { ** FTS5_NOT (nChild, apChild valid) ** FTS5_STRING (pNear valid) ** FTS5_TERM (pNear valid) +** +** iHeight: +** Distance from this node to furthest leaf. This is always 0 for nodes +** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one +** greater than the largest child value. */ struct Fts5ExprNode { int eType; /* Node type */ int bEof; /* True at EOF */ int bNomatch; /* True if entry is not a match */ + int iHeight; /* Distance to tree leaf nodes */ /* Next method for this node. */ int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64); @@ -227088,6 +228983,31 @@ struct Fts5Parse { int bPhraseToAnd; /* Convert "a+b" to "a AND b" */ }; +/* +** Check that the Fts5ExprNode.iHeight variables are set correctly in +** the expression tree passed as the only argument. +*/ +#ifndef NDEBUG +static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){ + if( rc==SQLITE_OK ){ + if( p->eType==FTS5_TERM || p->eType==FTS5_STRING || p->eType==0 ){ + assert( p->iHeight==0 ); + }else{ + int ii; + int iMaxChild = 0; + for(ii=0; iinChild; ii++){ + Fts5ExprNode *pChild = p->apChild[ii]; + iMaxChild = MAX(iMaxChild, pChild->iHeight); + assert_expr_depth_ok(SQLITE_OK, pChild); + } + assert( p->iHeight==iMaxChild+1 ); + } + } +} +#else +# define assert_expr_depth_ok(rc, p) +#endif + static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ va_list ap; va_start(ap, zFmt); @@ -227202,6 +229122,8 @@ static int sqlite3Fts5ExprNew( }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + assert_expr_depth_ok(sParse.rc, sParse.pExpr); + /* If the LHS of the MATCH expression was a user column, apply the ** implicit column-filter. */ if( iColnCol && sParse.pExpr && sParse.rc==SQLITE_OK ){ @@ -227364,7 +229286,7 @@ static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ Fts5Parse sParse; memset(&sParse, 0, sizeof(sParse)); - if( *pp1 ){ + if( *pp1 && p2 ){ Fts5Expr *p1 = *pp1; int nPhrase = p1->nPhrase + p2->nPhrase; @@ -227389,7 +229311,7 @@ static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ } sqlite3_free(p2->apExprPhrase); sqlite3_free(p2); - }else{ + }else if( p2 ){ *pp1 = p2; } @@ -229163,6 +231085,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ } static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ + int ii = p->nChild; if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); @@ -229171,6 +231094,9 @@ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ }else{ p->apChild[p->nChild++] = pSub; } + for( ; iinChild; ii++){ + p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1); + } } /* @@ -229201,6 +231127,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd( if( pRet ){ pRet->eType = FTS5_AND; pRet->nChild = nTerm; + pRet->iHeight = 1; fts5ExprAssignXNext(pRet); pParse->nPhrase--; for(ii=0; iiiHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){ + sqlite3Fts5ParseError(pParse, + "fts5 expression tree is too large (maximum depth %d)", + SQLITE_FTS5_MAX_EXPR_DEPTH + ); + sqlite3_free(pRet); + pRet = 0; + } } } } @@ -230906,6 +232841,8 @@ struct Fts5Index { sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ + sqlite3_stmt *pDeleteFromIdx; + sqlite3_stmt *pDataVersion; i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ @@ -230998,9 +232935,6 @@ struct Fts5CResult { ** iLeafOffset: ** Byte offset within the current leaf that is the first byte of the ** position list data (one byte passed the position-list size field). -** rowid field of the current entry. Usually this is the size field of the -** position list data. The exception is if the rowid for the current entry -** is the last thing on the leaf page. ** ** pLeaf: ** Buffer containing current leaf page data. Set to NULL at EOF. @@ -231559,6 +233493,7 @@ static int fts5StructureDecode( rc = FTS5_CORRUPT; break; } + assert( pSeg!=0 ); i += fts5GetVarint32(&pData[i], pSeg->iSegid); i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); @@ -231589,6 +233524,7 @@ static int fts5StructureDecode( */ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ fts5StructureMakeWritable(pRc, ppStruct); + assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK ); if( *pRc==SQLITE_OK ){ Fts5Structure *pStruct = *ppStruct; int nLevel = pStruct->nLevel; @@ -232047,42 +233983,25 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ pLvl->bEof = 1; }else{ u8 *a = pLvl->pData->p; - i64 iVal; - int iLimit; - int ii; - int nZero = 0; - - /* Currently iOff points to the first byte of a varint. This block - ** decrements iOff until it points to the first byte of the previous - ** varint. Taking care not to read any memory locations that occur - ** before the buffer in memory. */ - iLimit = (iOff>9 ? iOff-9 : 0); - for(iOff--; iOff>iLimit; iOff--){ - if( (a[iOff-1] & 0x80)==0 ) break; - } - - fts5GetVarint(&a[iOff], (u64*)&iVal); - pLvl->iRowid -= iVal; - pLvl->iLeafPgno--; - - /* Skip backwards past any 0x00 varints. */ - for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){ - nZero++; - } - if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){ - /* The byte immediately before the last 0x00 byte has the 0x80 bit - ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80 - ** bytes before a[ii]. */ - int bZero = 0; /* True if last 0x00 counts */ - if( (ii-8)>=pLvl->iFirstOff ){ - int j; - for(j=1; j<=8 && (a[ii-j] & 0x80); j++); - bZero = (j>8); + + pLvl->iOff = 0; + fts5DlidxLvlNext(pLvl); + while( 1 ){ + int nZero = 0; + int ii = pLvl->iOff; + u64 delta = 0; + + while( a[ii]==0 ){ + nZero++; + ii++; } - if( bZero==0 ) nZero--; + ii += sqlite3Fts5GetVarint(&a[ii], &delta); + + if( ii>=iOff ) break; + pLvl->iLeafPgno += nZero+1; + pLvl->iRowid += delta; + pLvl->iOff = ii; } - pLvl->iLeafPgno -= nZero; - pLvl->iOff = iOff - nZero; } return pLvl->bEof; @@ -232278,7 +234197,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ i64 iOff = pIter->iLeafOffset; ASSERT_SZLEAF_OK(pIter->pLeaf); - if( iOff>=pIter->pLeaf->szLeaf ){ + while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; @@ -232377,10 +234296,12 @@ static void fts5SegIterInit( fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; - fts5SegIterNextPage(p, pIter); + do { + fts5SegIterNextPage(p, pIter); + }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 ); } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && pIter->pLeaf ){ pIter->iLeafOffset = 4; assert( pIter->pLeaf!=0 ); assert_nc( pIter->pLeaf->nn>4 ); @@ -232574,7 +234495,7 @@ static void fts5SegIterNext_None( iOff = pIter->iLeafOffset; /* Next entry is on the next page */ - if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( p->rc || pIter->pLeaf==0 ) return; pIter->iRowid = 0; @@ -232767,7 +234688,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ Fts5Data *pLast = 0; int pgnoLast = 0; - if( pDlidx ){ + if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){ int iSegid = pIter->pSeg->iSegid; pgnoLast = fts5DlidxIterPgno(pDlidx); pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); @@ -233328,7 +235249,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ /* ** Move the seg-iter so that it points to the first rowid on page iLeafPgno. -** It is an error if leaf iLeafPgno does not exist or contains no rowids. +** It is an error if leaf iLeafPgno does not exist. Unless the db is +** a 'secure-delete' db, if it contains no rowids then this is also an error. */ static void fts5SegIterGotoPage( Fts5Index *p, /* FTS5 backend object */ @@ -233343,21 +235265,23 @@ static void fts5SegIterGotoPage( fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; pIter->iLeafPgno = iLeafPgno-1; - fts5SegIterNextPage(p, pIter); - assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); - if( p->rc==SQLITE_OK && ALWAYS(pIter->pLeaf!=0) ){ + while( p->rc==SQLITE_OK ){ int iOff; - u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->szLeaf; - + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) break; iOff = fts5LeafFirstRowidOff(pIter->pLeaf); - if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; - }else{ - iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; - fts5SegIterLoadNPos(p, pIter); + if( iOff>0 ){ + u8 *a = pIter->pLeaf->p; + int n = pIter->pLeaf->szLeaf; + if( iOff<4 || iOff>=n ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + fts5SegIterLoadNPos(p, pIter); + } + break; } } } @@ -234072,7 +235996,7 @@ static void fts5MultiIterNew( if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; - nSeg += (p->pHash ? 1 : 0); + nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH)); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } @@ -234093,7 +236017,7 @@ static void fts5MultiIterNew( if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; - if( p->pHash ){ + if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); @@ -234848,7 +236772,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); - fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]); + fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]); if( p->rc==SQLITE_OK ){ /* Set the szLeaf field */ fts5PutU16(&buf.p[2], (u16)buf.n); @@ -235126,16 +237050,16 @@ static void fts5IndexCrisismerge( ){ const int nCrisis = p->pConfig->nCrisisMerge; Fts5Structure *pStruct = *ppStruct; - int iLvl = 0; - - assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); - while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ - fts5IndexMergeLevel(p, &pStruct, iLvl, 0); - assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); - fts5StructurePromote(p, iLvl+1, pStruct); - iLvl++; + if( pStruct && pStruct->nLevel>0 ){ + int iLvl = 0; + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; } - *ppStruct = pStruct; } static int fts5IndexReturn(Fts5Index *p){ @@ -235169,6 +237093,413 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ return ret; } +/* +** Execute the SQL statement: +** +** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno); +** +** This is used when a secure-delete operation removes the last term +** from a segment leaf page. In that case the %_idx entry is removed +** too. This is done to ensure that if all instances of a token are +** removed from an fts5 database in secure-delete mode, no trace of +** the token itself remains in the database. +*/ +static void fts5SecureDeleteIdxEntry( + Fts5Index *p, /* FTS5 backend object */ + int iSegid, /* Id of segment to delete entry for */ + int iPgno /* Page number within segment */ +){ + if( iPgno!=1 ){ + assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE ); + if( p->pDeleteFromIdx==0 ){ + fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)", + p->pConfig->zDb, p->pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid); + sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno); + sqlite3_step(p->pDeleteFromIdx); + p->rc = sqlite3_reset(p->pDeleteFromIdx); + } + } +} + +/* +** This is called when a secure-delete operation removes a position-list +** that overflows onto segment page iPgno of segment pSeg. This function +** rewrites node iPgno, and possibly one or more of its right-hand peers, +** to remove this portion of the position list. +** +** Output variable (*pbLastInDoclist) is set to true if the position-list +** removed is followed by a new term or the end-of-segment, or false if +** it is followed by another rowid/position list. +*/ +static void fts5SecureDeleteOverflow( + Fts5Index *p, + Fts5StructureSegment *pSeg, + int iPgno, + int *pbLastInDoclist +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int pgno; + Fts5Data *pLeaf = 0; + assert( iPgno!=1 ); + + *pbLastInDoclist = 1; + for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){ + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + int iNext = 0; + u8 *aPg = 0; + + pLeaf = fts5DataRead(p, iRowid); + if( pLeaf==0 ) break; + aPg = pLeaf->p; + + iNext = fts5GetU16(&aPg[0]); + if( iNext!=0 ){ + *pbLastInDoclist = 0; + } + if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){ + fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext); + } + + if( iNext==0 ){ + /* The page contains no terms or rowids. Replace it with an empty + ** page and move on to the right-hand peer. */ + const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; + assert_nc( bDetailNone==0 || pLeaf->nn==4 ); + if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty)); + fts5DataRelease(pLeaf); + pLeaf = 0; + }else if( bDetailNone ){ + break; + }else if( iNext>=pLeaf->szLeaf || iNext<4 ){ + p->rc = FTS5_CORRUPT; + break; + }else{ + int nShift = iNext - 4; + int nPg; + + int nIdx = 0; + u8 *aIdx = 0; + + /* Unless the current page footer is 0 bytes in size (in which case + ** the new page footer will be as well), allocate and populate a + ** buffer containing the new page footer. Set stack variables aIdx + ** and nIdx accordingly. */ + if( pLeaf->nn>pLeaf->szLeaf ){ + int iFirst = 0; + int i1 = pLeaf->szLeaf; + int i2 = 0; + + aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); + if( aIdx==0 ) break; + i1 += fts5GetVarint32(&aPg[i1], iFirst); + i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); + if( i1nn ){ + memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); + i2 += (pLeaf->nn-i1); + } + nIdx = i2; + } + + /* Modify the contents of buffer aPg[]. Set nPg to the new size + ** in bytes. The new page is always smaller than the old. */ + nPg = pLeaf->szLeaf - nShift; + memmove(&aPg[4], &aPg[4+nShift], nPg-4); + fts5PutU16(&aPg[2], nPg); + if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4); + if( nIdx>0 ){ + memcpy(&aPg[nPg], aIdx, nIdx); + nPg += nIdx; + } + sqlite3_free(aIdx); + + /* Write the new page to disk and exit the loop */ + assert( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, iRowid, aPg, nPg); + break; + } + } + fts5DataRelease(pLeaf); +} + +/* +** Completely remove the entry that pSeg currently points to from +** the database. +*/ +static void fts5DoSecureDelete( + Fts5Index *p, + Fts5SegIter *pSeg +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int iSegid = pSeg->pSeg->iSegid; + u8 *aPg = pSeg->pLeaf->p; + int nPg = pSeg->pLeaf->nn; + int iPgIdx = pSeg->pLeaf->szLeaf; + + u64 iDelta = 0; + u64 iNextDelta = 0; + int iNextOff = 0; + int iOff = 0; + int nIdx = 0; + u8 *aIdx = 0; + int bLastInDoclist = 0; + int iIdx = 0; + int iStart = 0; + int iKeyOff = 0; + int iPrevKeyOff = 0; + int iDelKeyOff = 0; /* Offset of deleted key, if any */ + + nIdx = nPg-iPgIdx; + aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + if( p->rc ) return; + memcpy(aIdx, &aPg[iPgIdx], nIdx); + + /* At this point segment iterator pSeg points to the entry + ** this function should remove from the b-tree segment. + ** + ** In detail=full or detail=column mode, pSeg->iLeafOffset is the + ** offset of the first byte in the position-list for the entry to + ** remove. Immediately before this comes two varints that will also + ** need to be removed: + ** + ** + the rowid or delta rowid value for the entry, and + ** + the size of the position list in bytes. + ** + ** Or, in detail=none mode, there is a single varint prior to + ** pSeg->iLeafOffset - the rowid or delta rowid value. + ** + ** This block sets the following variables: + ** + ** iStart: + ** iDelta: + */ + { + int iSOP; + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ + iStart = pSeg->iTermLeafOffset; + }else{ + iStart = fts5GetU16(&aPg[0]); + } + + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + assert_nc( iSOP<=pSeg->iLeafOffset ); + + if( bDetailNone ){ + while( iSOPiLeafOffset ){ + if( aPg[iSOP]==0x00 ) iSOP++; + if( aPg[iSOP]==0x00 ) iSOP++; + iStart = iSOP; + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + } + + iNextOff = iSOP; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + + }else{ + int nPos = 0; + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + while( iSOPiLeafOffset ){ + iStart = iSOP + (nPos/2); + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + } + assert_nc( iSOP==pSeg->iLeafOffset ); + iNextOff = pSeg->iLeafOffset + pSeg->nPos; + } + } + + iOff = iStart; + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + }else{ + /* Set bLastInDoclist to true if the entry being removed is the last + ** in its doclist. */ + for(iIdx=0, iKeyOff=0; iIdxiTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno + ){ + /* The entry being removed was the only position list in its + ** doclist. Therefore the term needs to be removed as well. */ + int iKey = 0; + for(iIdx=0, iKeyOff=0; iIdx(u32)iStart ) break; + iKeyOff += iVal; + } + + iDelKeyOff = iOff = iKeyOff; + if( iNextOff!=iPgIdx ){ + int nPrefix = 0; + int nSuffix = 0; + int nPrefix2 = 0; + int nSuffix2 = 0; + + iDelKeyOff = iNextOff; + iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + + if( iKey!=1 ){ + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + } + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + + nPrefix = MIN(nPrefix, nPrefix2); + nSuffix = (nPrefix2 + nSuffix2) - nPrefix; + + if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + p->rc = FTS5_CORRUPT; + }else{ + if( iKey!=1 ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + } + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); + if( nPrefix2>nPrefix ){ + memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); + iOff += (nPrefix2-nPrefix); + } + memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); + iOff += nSuffix2; + iNextOff += nSuffix2; + } + } + }else if( iStart==4 ){ + int iPgno; + + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; + + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } + } + fts5DataRelease(pTerm); + } + } + + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; + int nShift = 0; + + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= (iNextOff - iOff); + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); + + nShift = iNextOff - iOff; + for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdxiOff ){ + iKeyOff -= nShift; + nShift = 0; + } + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff); + iPrevKeyOff = iKeyOff; + } + } + + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + } + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg); + } + sqlite3_free(aIdx); +} + +/* +** This is called as part of flushing a delete to disk in 'secure-delete' +** mode. It edits the segments within the database described by argument +** pStruct to remove the entries for term zTerm, rowid iRowid. +*/ +static void fts5FlushSecureDelete( + Fts5Index *p, + Fts5Structure *pStruct, + const char *zTerm, + i64 iRowid +){ + const int f = FTS5INDEX_QUERY_SKIPHASH; + int nTerm = (int)strlen(zTerm); + Fts5Iter *pIter = 0; /* Used to find term instance */ + + fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); + if( fts5MultiIterEof(p, pIter)==0 ){ + i64 iThis = fts5MultiIterRowid(pIter); + if( iThisrc==SQLITE_OK + && fts5MultiIterEof(p, pIter)==0 + && iRowid==fts5MultiIterRowid(pIter) + ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + fts5DoSecureDelete(p, pSeg); + } + } + + fts5MultiIterFree(pIter); +} + + /* ** Flush the contents of in-memory hash table iHash to a new level-0 ** segment on disk. Also update the corresponding structure record. @@ -235191,6 +237522,7 @@ static void fts5FlushOneHash(Fts5Index *p){ if( iSegid ){ const int pgsz = p->pConfig->pgsz; int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; Fts5StructureSegment *pSeg; /* New segment within pStruct */ Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ @@ -235213,40 +237545,77 @@ static void fts5FlushOneHash(Fts5Index *p){ } while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ const u8 *pDoclist; /* Pointer to doclist for this term */ int nDoclist; /* Size of doclist in bytes */ - /* Write the term for this entry to disk. */ + /* Get the term and doclist for this entry. */ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); - if( p->rc!=SQLITE_OK ) break; + nTerm = (int)strlen(zTerm); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } - assert( writer.bFirstRowidInPage==0 ); - if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); }else{ + int bTermWritten = !bSecureDelete; i64 iRowid = 0; - u64 iDelta = 0; + i64 iPrev = 0; int iOff = 0; /* The entire doclist will not fit on this leaf. The following ** loop iterates through the poslists that make up the current ** doclist. */ while( p->rc==SQLITE_OK && iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ + iOff++; + continue; + } + } + } + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + bTermWritten = 1; + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); + } + if( writer.bFirstRowidInPage ){ fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); writer.bFirstRowidInPage = 0; fts5WriteDlidxAppend(p, &writer, iRowid); - if( p->rc!=SQLITE_OK ) break; }else{ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev); } + if( p->rc!=SQLITE_OK ) break; assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; if( eDetail==FTS5_DETAIL_NONE ){ if( iOffnLevel==0 ){ - fts5StructureAddLevel(&p->rc, &pStruct); - } - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); - if( p->rc==SQLITE_OK ){ - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; - pStruct->nSegment++; + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); } - fts5StructurePromote(p, 0, pStruct); } fts5IndexAutomerge(p, &pStruct, pgnoLast); @@ -236059,6 +238431,7 @@ static int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); sqlite3_finalize(p->pDataVersion); + sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); @@ -236689,6 +239062,7 @@ static void fts5IndexIntegrityCheckSegment( Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5Config *pConfig = p->pConfig; + int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE); sqlite3_stmt *pStmt = 0; int rc2; int iIdxPrevLeaf = pSeg->pgnoFirst-1; @@ -236724,7 +239098,19 @@ static void fts5IndexIntegrityCheckSegment( ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ if( pLeaf->nn<=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + + if( nIdxTerm==0 + && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE + && pLeaf->nn==pLeaf->szLeaf + && pLeaf->nn==4 + ){ + /* special case - the very first page in a segment keeps its %_idx + ** entry even if all the terms are removed from it by secure-delete + ** operations. */ + }else{ + p->rc = FTS5_CORRUPT; + } + }else{ int iOff; /* Offset of first term on leaf */ int iRowidOff; /* Offset of first rowid on leaf */ @@ -236788,9 +239174,12 @@ static void fts5IndexIntegrityCheckSegment( ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; - }else{ + }else if( bSecureDelete==0 || iRowidOff>0 ){ + i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); - if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; + if( iRowidrc = FTS5_CORRUPT; + } } fts5DataRelease(pLeaf); } @@ -239052,6 +241441,8 @@ static int fts5UpdateMethod( Fts5Config *pConfig = pTab->p.pConfig; int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ + int bUpdateOrDelete = 0; + /* A transaction must be open when this is called. */ assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); @@ -239062,6 +241453,11 @@ static int fts5UpdateMethod( || sqlite3_value_type(apVal[0])==SQLITE_NULL ); assert( pTab->p.pConfig->pzErrmsg==0 ); + if( pConfig->pgsz==0 ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) return rc; + } + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; /* Put any active cursors into REQUIRE_SEEK state. */ @@ -239114,6 +241510,7 @@ static int fts5UpdateMethod( else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); + bUpdateOrDelete = 1; } /* INSERT or UPDATE */ @@ -239129,6 +241526,7 @@ static int fts5UpdateMethod( if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + bUpdateOrDelete = 1; } fts5StorageInsert(&rc, pTab, apVal, pRowid); } @@ -239157,10 +241555,24 @@ static int fts5UpdateMethod( rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); fts5StorageInsert(&rc, pTab, apVal, pRowid); } + bUpdateOrDelete = 1; } } } + if( rc==SQLITE_OK + && bUpdateOrDelete + && pConfig->bSecureDelete + && pConfig->iVersion==FTS5_CURRENT_VERSION + ){ + rc = sqlite3Fts5StorageConfigValue( + pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE + ); + if( rc==SQLITE_OK ){ + pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; + } + } + pTab->p.pConfig->pzErrmsg = 0; return rc; } @@ -240020,6 +242432,7 @@ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); + pTab->p.pConfig->pgsz = 0; return sqlite3Fts5StorageRollback(pTab->pStorage); } @@ -240222,7 +242635,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2023-03-10 12:13:52 20399f3eda5ec249d147ba9e48da6e87f969d7966a9a896764ca437ff7e737ff", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0", -1, SQLITE_TRANSIENT); } /* diff --git a/src/database/sqlite3.h b/src/database/sqlite3.h index 0a3c56c97..48effe202 100644 --- a/src/database/sqlite3.h +++ b/src/database/sqlite3.h @@ -146,9 +146,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.41.1" -#define SQLITE_VERSION_NUMBER 3041001 -#define SQLITE_SOURCE_ID "2023-03-10 12:13:52 20399f3eda5ec249d147ba9e48da6e87f969d7966a9a896764ca437ff7e737ff" +#define SQLITE_VERSION "3.42.0" +#define SQLITE_VERSION_NUMBER 3042000 +#define SQLITE_SOURCE_ID "2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -1655,20 +1655,23 @@ SQLITE_API int sqlite3_os_end(void); ** must ensure that no other SQLite interfaces are invoked by other ** threads while sqlite3_config() is running. ** -** The sqlite3_config() interface -** may only be invoked prior to library initialization using -** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. -** ^If sqlite3_config() is called after [sqlite3_initialize()] and before -** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. -** Note, however, that ^sqlite3_config() can be called as part of the -** implementation of an application-defined [sqlite3_os_init()]. -** ** The first argument to sqlite3_config() is an integer ** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments ** vary depending on the [configuration option] ** in the first argument. ** +** For most configuration options, the sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** The exceptional configuration options that may be invoked at any time +** are called "anytime configuration options". +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] with a first argument that is not an anytime +** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. @@ -1776,6 +1779,23 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. ** +** Most of the configuration options for sqlite3_config() +** will only work if invoked prior to [sqlite3_initialize()] or after +** [sqlite3_shutdown()]. The few exceptions to this rule are called +** "anytime configuration options". +** ^Calling [sqlite3_config()] with a first argument that is not an +** anytime configuration option in between calls to [sqlite3_initialize()] and +** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE. +** +** The set of anytime configuration options can change (by insertions +** and/or deletions) from one release of SQLite to the next. +** As of SQLite version 3.42.0, the complete set of anytime configuration +** options is: +**
    +**
  • SQLITE_CONFIG_LOG +**
  • SQLITE_CONFIG_PCACHE_HDRSZ +**
+** ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications ** should check the return code from [sqlite3_config()] to make sure that @@ -2122,28 +2142,28 @@ struct sqlite3_mem_methods { ** compile-time option is not set, then the default maximum is 1073741824. ** */ -#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ -#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ -#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ -#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ -#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ -#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ -#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ -#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ -#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ -/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ -#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ -#define SQLITE_CONFIG_PCACHE 14 /* no-op */ -#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ -#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ -#define SQLITE_CONFIG_URI 17 /* int */ -#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ -#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ -#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ -#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ @@ -2378,7 +2398,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_DQS_DML]] -**
SQLITE_DBCONFIG_DQS_DML +**
SQLITE_DBCONFIG_DQS_DML
**
The SQLITE_DBCONFIG_DQS_DML option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DML statements ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The @@ -2387,7 +2407,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_DQS_DDL]] -**
SQLITE_DBCONFIG_DQS_DDL +**
SQLITE_DBCONFIG_DQS_DDL
**
The SQLITE_DBCONFIG_DQS option activates or deactivates ** the legacy [double-quoted string literal] misfeature for DDL statements, ** such as CREATE TABLE and CREATE INDEX. The @@ -2396,7 +2416,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] -**
SQLITE_DBCONFIG_TRUSTED_SCHEMA +**
SQLITE_DBCONFIG_TRUSTED_SCHEMA
**
The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to ** assume that database schemas are untainted by malicious content. ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite @@ -2416,7 +2436,7 @@ struct sqlite3_mem_methods { **
** ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] -**
SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +**
SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
**
The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates ** the legacy file format flag. When activated, this flag causes all newly ** created database file to have a schema format version number (the 4-byte @@ -2425,7 +2445,7 @@ struct sqlite3_mem_methods { ** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, ** newly created databases are generally not understandable by SQLite versions ** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there -** is now scarcely any need to generated database files that are compatible +** is now scarcely any need to generate database files that are compatible ** all the way back to version 3.0.0, and so this setting is of little ** practical use, but is provided so that SQLite can continue to claim the ** ability to generate new database files that are compatible with version @@ -2436,6 +2456,38 @@ struct sqlite3_mem_methods { ** not considered a bug since SQLite versions 3.3.0 and earlier do not support ** either generated columns or decending indexes. **
+** +** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] +**
SQLITE_DBCONFIG_STMT_SCANSTATUS
+**
The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is set (collection of statistics is enabled) +** by default. This option takes two arguments: an integer and a pointer to +** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the statement scanstatus option. If the second argument +** is not NULL, then the value of the statement scanstatus setting after +** processing the first argument is written into the integer that the second +** argument points to. +**
+** +** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] +**
SQLITE_DBCONFIG_REVERSE_SCANORDER
+**
The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order +** in which tables and indexes are scanned so that the scans start at the end +** and work toward the beginning rather than starting at the beginning and +** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** two arguments which are an integer and a pointer to an integer. The first +** argument is 1, 0, or -1 to enable, disable, or leave unchanged the +** reverse scan order flag, respectively. If the second argument is not NULL, +** then 0 or 1 is written into the integer that the second argument points to +** depending on if the reverse scan order flag is set after processing the +** first argument. +**
+** ** */ #define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ @@ -2456,7 +2508,9 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ #define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ +#define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -6201,6 +6255,13 @@ SQLITE_API void sqlite3_activate_cerod( ** of the default VFS is not implemented correctly, or not implemented at ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. +** +** If a negative argument is passed to sqlite3_sleep() the results vary by +** VFS and operating system. Some system treat a negative argument as an +** instruction to sleep forever. Others understand it to mean do not sleep +** at all. ^In SQLite version 3.42.0 and later, a negative +** argument passed into sqlite3_sleep() is changed to zero before it is relayed +** down into the xSleep method of the VFS. */ SQLITE_API int sqlite3_sleep(int); @@ -7828,9 +7889,9 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); ** is undefined if the mutex is not currently entered by the ** calling thread or is not currently allocated. ** -** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or -** sqlite3_mutex_leave() is a NULL pointer, then all three routines -** behave as no-ops. +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), +** sqlite3_mutex_leave(), or sqlite3_mutex_free() is a NULL pointer, +** then any of the four routines behaves as a no-op. ** ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. */ @@ -9564,18 +9625,28 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** [[SQLITE_VTAB_INNOCUOUS]]
SQLITE_VTAB_INNOCUOUS
**
Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the -** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** the [xConnect] or [xCreate] methods of a [virtual table] implementation ** identify that virtual table as being safe to use from within triggers ** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the ** virtual table can do no serious harm even if it is controlled by a ** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS ** flag unless absolutely necessary. **
+** +** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]
SQLITE_VTAB_USES_ALL_SCHEMAS
+**
Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** instruct the query planner to begin at least a read transaction on +** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the +** virtual table is used. +**
** */ #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 #define SQLITE_VTAB_INNOCUOUS 2 #define SQLITE_VTAB_DIRECTONLY 3 +#define SQLITE_VTAB_USES_ALL_SCHEMAS 4 /* ** CAPI3REF: Determine The Virtual Table Conflict Policy @@ -10750,16 +10821,20 @@ SQLITE_API int sqlite3session_create( SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); /* -** CAPIREF: Conigure a Session Object +** CAPI3REF: Configure a Session Object ** METHOD: sqlite3_session ** ** This method is used to configure a session object after it has been -** created. At present the only valid value for the second parameter is -** [SQLITE_SESSION_OBJCONFIG_SIZE]. +** created. At present the only valid values for the second parameter are +** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID]. ** -** Arguments for sqlite3session_object_config() +*/ +SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); + +/* +** CAPI3REF: Options for sqlite3session_object_config ** -** The following values may passed as the the 4th parameter to +** The following values may passed as the the 2nd parameter to ** sqlite3session_object_config(). ** **
SQLITE_SESSION_OBJCONFIG_SIZE
@@ -10775,12 +10850,21 @@ SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); ** ** It is an error (SQLITE_MISUSE) to attempt to modify this setting after ** the first table has been attached to the session object. +** +**
SQLITE_SESSION_OBJCONFIG_ROWID
+** This option is used to set, clear or query the flag that enables +** collection of data for tables with no explicit PRIMARY KEY. +** +** Normally, tables with no explicit PRIMARY KEY are simply ignored +** by the sessions module. However, if this flag is set, it behaves +** as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted +** as their leftmost columns. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. */ -SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); - -/* -*/ -#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_ROWID 2 /* ** CAPI3REF: Enable Or Disable A Session Object @@ -11913,9 +11997,23 @@ SQLITE_API int sqlite3changeset_apply_v2( ** Invert the changeset before applying it. This is equivalent to inverting ** a changeset using sqlite3changeset_invert() before applying it. It is ** an error to specify this flag with a patchset. +** +**
SQLITE_CHANGESETAPPLY_IGNORENOOP
+** Do not invoke the conflict handler callback for any changes that +** would not actually modify the database even if they were applied. +** Specifically, this means that the conflict handler is not invoked +** for: +**
    +**
  • a delete change if the row being deleted cannot be found, +**
  • an update change if the modified fields are already set to +** their new values in the conflicting row, or +**
  • an insert change if all fields of the conflicting row match +** the row being inserted. +**
*/ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 #define SQLITE_CHANGESETAPPLY_INVERT 0x0002 +#define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 /* ** CAPI3REF: Constants Passed To The Conflict Handler From c33643408a5abf95cb884f5d73453fd42d71ff7c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 18 May 2023 10:17:29 +0200 Subject: [PATCH 59/78] Apply Pi-hole specific patches Signed-off-by: DL6ER --- src/database/shell.c | 5 ++++- src/database/sqlite3.c | 4 +--- test/test_suite.bats | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/database/shell.c b/src/database/shell.c index 647a21422..dba3e9701 100644 --- a/src/database/shell.c +++ b/src/database/shell.c @@ -127,6 +127,8 @@ typedef unsigned char u8; #endif #include #include +// print_FTL_version() +#include "../log.h" #if !defined(_WIN32) && !defined(WIN32) # include @@ -27271,7 +27273,7 @@ static void sayAbnormalExit(void){ #endif #if SQLITE_SHELL_IS_UTF8 -int SQLITE_CDECL main(int argc, char **argv){ +int SQLITE_CDECL sqlite3_shell_main(int argc, char **argv){ #else int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char **argv; @@ -27829,6 +27831,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){ char *zHome; char *zHistory; int nHistory; + print_FTL_version(); printf( "SQLite version %s %.19s\n" /*extra-version-info*/ "Enter \".help\" for usage hints.\n", diff --git a/src/database/sqlite3.c b/src/database/sqlite3.c index dd3b5c575..16c976950 100644 --- a/src/database/sqlite3.c +++ b/src/database/sqlite3.c @@ -176379,8 +176379,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** Return a static string containing the name corresponding to the error code ** specified in the argument. */ -#if defined(SQLITE_NEED_ERR_NAME) -SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ +SQLITE_API const char *sqlite3ErrName(int rc){ const char *zName = 0; int i, origRc = rc; for(i=0; i<2 && zName==0; i++, rc &= 0xff){ @@ -176486,7 +176485,6 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ } return zName; } -#endif /* ** Return a static string that describes the kind of error specified in the diff --git a/test/test_suite.bats b/test/test_suite.bats index cc340ffb2..700263c1a 100644 --- a/test/test_suite.bats +++ b/test/test_suite.bats @@ -1356,7 +1356,7 @@ @test "Embedded SQLite3 shell available and functional" { run bash -c './pihole-FTL sqlite3 -help' printf "%s\n" "${lines[@]}" - [[ ${lines[0]} == "Usage: sqlite3 [OPTIONS] FILENAME [SQL]" ]] + [[ ${lines[0]} == "Usage: sqlite3 [OPTIONS] [FILENAME [SQL]]" ]] } @test "Embedded SQLite3 shell is called for .db file" { From 31a90da5ca7464611b5e406613414488c9457ce3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 May 2023 13:57:20 +0200 Subject: [PATCH 60/78] Align % in reply rate column Signed-off-by: DL6ER --- src/tools/arp-scan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 14f9c9f01..46e3496f3 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -575,7 +575,7 @@ static void print_results(struct thread_data *thread_data) inet_ntop(AF_INET, &ip, thread_data->ipstr, INET_ADDRSTRLEN); // Print MAC address - printf("%-16s %-16s %-24s %02x:%02x:%02x:%02x:%02x:%02x %u %%\n", + printf("%-16s %-16s %-24s %02x:%02x:%02x:%02x:%02x:%02x %3u %%\n", thread_data->ipstr, thread_data->iface, get_hostname(&ip), mac[0], mac[1], mac[2], mac[3], mac[4], mac[5], From 04a7633a32c84715beeeaa11e37ed8e013305a76 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 May 2023 14:40:27 +0200 Subject: [PATCH 61/78] Add capabilities check for feature dhcp-discover in the same way we already have it for arp-scan Signed-off-by: DL6ER --- src/tools/dhcp-discover.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 5e2613eaa..14e6b5e21 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -17,6 +17,8 @@ #include "log.h" // read_FTLconf() #include "config.h" +// check_capability() +#include "capabilities.h" #include // SIOCGIFHWADDR @@ -691,6 +693,14 @@ static void *dhcp_discover_iface(void *args) int run_dhcp_discover(void) { + // Check if we are capable of binding to port 67 (DHCP) + // DHCP uses normal UDP datagrams, so we cdon't need CAP_NET_RAW + if(!check_capability(CAP_NET_BIND_SERVICE)) + { + puts("Error: Insufficient permissions or capabilities (needs CAP_NET_BIND_SERVICE). Try running as root (sudo)"); + return EXIT_FAILURE; + } + // Disable terminal output during config config file parsing log_ctrl(false, false); // Process pihole-FTL.conf to get gravity.db From aa8821adbcdcce65af717b5fa45268d507a69a24 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 May 2023 22:19:53 +0200 Subject: [PATCH 62/78] Modify logging in such a way that concurrent printing by the involved is prevented and add better error reporting when sending to interfaces is not working due to an error Signed-off-by: DL6ER --- src/syscalls/sendto.c | 3 +- src/tools/dhcp-discover.c | 214 +++++++++++++++++++++----------------- 2 files changed, 121 insertions(+), 96 deletions(-) diff --git a/src/syscalls/sendto.c b/src/syscalls/sendto.c index 846c571e7..c6bf5c01c 100644 --- a/src/syscalls/sendto.c +++ b/src/syscalls/sendto.c @@ -34,7 +34,8 @@ ssize_t FTLsendto(int sockfd, void *buf, size_t len, int flags, const struct soc // Final error checking (may have failed for some other reason then an // EINTR = interrupted system call), also ignore EPROTONOSUPPORT (ARP scanning) - if(ret < 0 && errno != EPROTONOSUPPORT) + // and EPERM + ENOKEY (DHCP probing) + if(ret < 0 && errno != EPROTONOSUPPORT && errno != EPERM && errno != ENOKEY) logg("WARN: Could not sendto() in %s() (%s:%i): %s", func, file, line, strerror(errno)); diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 14e6b5e21..18b415b3b 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -13,7 +13,7 @@ #undef __USE_XOPEN #include "FTL.h" #include "dhcp-discover.h" -// logg(), format_time() +// format_time() #include "log.h" // read_FTLconf() #include "config.h" @@ -60,6 +60,15 @@ // Global lock used by all threads static pthread_mutex_t lock; +static void __attribute__((format(gnu_printf, 1, 2))) printf_locked(const char *format, ...) +{ + va_list args; + va_start(args, format); + pthread_mutex_lock(&lock); + vprintf(format, args); + pthread_mutex_unlock(&lock); + va_end(args); +} extern const struct opttab_t { char *name; @@ -84,17 +93,17 @@ static int create_dhcp_socket(const char *iname) const int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); if(sock < 0) { - logg("Error: Could not create socket for interface %s!", iname); + printf_locked("Error: Could not create socket for interface %s!\n", iname); return -1; } #ifdef DEBUG - logg("DHCP socket: %d", sock); + printf_locked("DHCP socket: %d\n", sock); #endif // set the reuse address flag so we don't get errors when restarting if(setsockopt(sock,SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof(flag))<0) { - logg("Error: Could not set reuse address option on DHCP socket (%s)!", iname); + printf_locked("Error: Could not set reuse address option on DHCP socket (%s)!\n", iname); close(sock); return -1; } @@ -102,7 +111,7 @@ static int create_dhcp_socket(const char *iname) // set the broadcast option - we need this to listen to DHCP broadcast messages if(setsockopt(sock, SOL_SOCKET,SO_BROADCAST, (char *)&flag, sizeof flag) < 0) { - logg("Error: Could not set broadcast option on DHCP socket (%s)!", iname); + printf_locked("Error: Could not set broadcast option on DHCP socket (%s)!\n", iname); close(sock); return -1; } @@ -111,8 +120,8 @@ static int create_dhcp_socket(const char *iname) strncpy(interface.ifr_ifrn.ifrn_name, iname, IFNAMSIZ-1); if(setsockopt(sock,SOL_SOCKET, SO_BINDTODEVICE, (char *)&interface, sizeof(interface)) < 0) { - logg("Error: Could not bind socket to interface %s (%s)\n ---> Check your privileges (run with sudo)!\n", - iname, strerror(errno)); + printf_locked("Error: Could not bind socket to interface %s (%s)\n", + iname, strerror(errno)); close(sock); return -1; } @@ -120,8 +129,8 @@ static int create_dhcp_socket(const char *iname) // bind the socket if(bind(sock, (struct sockaddr *)&dhcp_socket, sizeof(dhcp_socket)) < 0) { - logg("Error: Could not bind to DHCP socket (interface %s, port %d, %s)\n ---> Check your privileges (run with sudo)!\n", - iname, DHCP_CLIENT_PORT, strerror(errno)); + printf_locked("Error: Could not bind to DHCP socket (interface %s, port %d, %s)\n", + iname, DHCP_CLIENT_PORT, strerror(errno)); close(sock); return -1; } @@ -137,16 +146,17 @@ int get_hardware_address(const int sock, const char *iname, unsigned char *mac) // try and grab hardware address of requested interface int ret = 0; - if((ret = ioctl(sock, SIOCGIFHWADDR, &ifr)) < 0){ - logg(" Error: Could not get hardware address of interface '%s' (socket %d, error: %s)", iname, sock, strerror(errno)); + if((ret = ioctl(sock, SIOCGIFHWADDR, &ifr)) < 0) + { + printf_locked(" Error: Could not get hardware address of interface '%s' (socket %d, error: %s)\n", iname, sock, strerror(errno)); return false; } memcpy(&mac[0], &ifr.ifr_hwaddr.sa_data, 6); #ifdef DEBUG - logg_sameline("Hardware address of this interface: "); + printf_locked("Hardware address of this interface: "); for (uint8_t i = 0; i < 6; ++i) - logg_sameline("%02x%s", mac[i], i < 5 ? ":" : ""); - logg(" "); + printf_locked("%02x%s", mac[i], i < 5 ? ":" : ""); + printf_locked("\n"); #endif return true; } @@ -224,20 +234,30 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i memset(&target.sin_zero, 0, sizeof(target.sin_zero)); #ifdef DEBUG - logg("Sending DHCPDISCOVER on interface %s:%s ... ", iface, inet_ntoa(target.sin_addr)); - logg("DHCPDISCOVER XID: %lu (0x%X)", (unsigned long) ntohl(discover_packet.xid), ntohl(discover_packet.xid)); - logg("DHCDISCOVER ciaddr: %s", inet_ntoa(discover_packet.ciaddr)); - logg("DHCDISCOVER yiaddr: %s", inet_ntoa(discover_packet.yiaddr)); - logg("DHCDISCOVER siaddr: %s", inet_ntoa(discover_packet.siaddr)); - logg("DHCDISCOVER giaddr: %s", inet_ntoa(discover_packet.giaddr)); + printf_locked("Sending DHCPDISCOVER on interface %s:%s ... \n", iface, inet_ntoa(target.sin_addr)); + printf_locked("DHCPDISCOVER XID: %lu (0x%X)\n", (unsigned long) ntohl(discover_packet.xid), ntohl(discover_packet.xid)); + printf_locked("DHCDISCOVER ciaddr: %s\n", inet_ntoa(discover_packet.ciaddr)); + printf_locked("DHCDISCOVER yiaddr: %s\n", inet_ntoa(discover_packet.yiaddr)); + printf_locked("DHCDISCOVER siaddr: %s\n", inet_ntoa(discover_packet.siaddr)); + printf_locked("DHCDISCOVER giaddr: %s\n", inet_ntoa(discover_packet.giaddr)); #endif // send the DHCPDISCOVER packet - const int bytes = sendto(sock, (char *)&discover_packet, sizeof(discover_packet), 0, (struct sockaddr *)&target,sizeof(target)); + const int bytes = sendto(sock, (char *)&discover_packet, sizeof(discover_packet), 0, (struct sockaddr *)&target, sizeof(target)); + if(bytes < 0) + { + // strerror() returns "Required key not available" for ENOKEY + // which is not helpful at all so we substitute a more + // meaningful error message for ENOKEY, which is returned when + // the network is unreachable + const char *error = errno == ENOKEY ? "Broadcast is unreachable" : strerror(errno); + printf_locked("Error: Could not send DHCPDISCOVER packet on interface %s: %s\n", iface, error); + return false; + } + #ifdef DEBUG - logg("Sent %d bytes", bytes); + printf_locked("Sent %d bytes\n", bytes); #endif - - return bytes > 0; + return true; } #ifdef TEST_OPT_249 @@ -274,7 +294,7 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack // Sanity check if(x >= MAX_DHCP_OPTIONS_LENGTH-2) { - logg(" OVERFLOWING DHCP OPTION (invalid size)"); + printf(" OVERFLOWING DHCP OPTION (invalid size)\n"); break; } @@ -284,12 +304,12 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack // get option length const uint8_t optlen = offer_packet->options[x++]; - logg_sameline(" "); + printf(" "); // Sanity check if(x + optlen > MAX_DHCP_OPTIONS_LENGTH) { - logg(" OVERFLOWING DHCP OPTION (invalid size)"); + printf(" OVERFLOWING DHCP OPTION (invalid size)\n"); break; } @@ -309,14 +329,14 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack struct in_addr addr_list = { 0 }; memcpy(&addr_list.s_addr, &offer_packet->options[x+n*4], sizeof(addr_list.s_addr)); if(n > 0) - logg_sameline(" "); + printf(" "); - logg("%s: %s", opttab[i].name, inet_ntoa(addr_list)); + printf("%s: %s\n", opttab[i].name, inet_ntoa(addr_list)); } // Special case: optlen == 0 if(optlen == 0) - logg("--- end of options ---"); + printf("--- end of options ---\n"); } else if(opttab[i].size & OT_NAME) { @@ -325,7 +345,7 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack // possible "(empty)" char buffer[4*optlen + 9]; binbuf_to_escaped_C_literal(&offer_packet->options[x], optlen, buffer, sizeof(buffer)); - logg("%s: \"%s\"", opttab[i].name, buffer); + printf("%s: \"%s\"\n", opttab[i].name, buffer); } else if(opttab[i].size & OT_TIME) { @@ -340,12 +360,12 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack optname = "rebinding-time"; // "T2" in dnsmasq-notation if(time == 0xFFFFFFFF) - logg("%s: Infinite", optname); + printf("%s: Infinite\n", optname); else { char buffer[42] = { 0 }; format_time(buffer, time, 0.0); - logg("%s: %lu (%s)", optname, (unsigned long)time, buffer); + printf("%s: %lu (%s)\n", optname, (unsigned long)time, buffer); } } else if(opttab[i].size & OT_DEC) @@ -355,31 +375,31 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack switch(offer_packet->options[x]) { case 1: - logg("Message type: DHCPDISCOVER (1)"); + printf("Message type: DHCPDISCOVER (1)\n"); break; case 2: - logg("Message type: DHCPOFFER (2)"); + printf("Message type: DHCPOFFER (2)\n"); break; case 3: - logg("Message type: DHCPREQUEST (3)"); + printf("Message type: DHCPREQUEST (3)\n"); break; case 4: - logg("Message type: DHCPDECLINE (4)"); + printf("Message type: DHCPDECLINE (4)\n"); break; case 5: - logg("Message type: DHCPACK (5)"); + printf("Message type: DHCPACK (5)\n"); break; case 6: - logg("Message type: DHCPNAK (6)"); + printf("Message type: DHCPNAK (6)\n"); break; case 7: - logg("Message type: DHCPRELEASE (7)"); + printf("Message type: DHCPRELEASE (7)\n"); break; case 8: - logg("Message type: DHCPINFORM (8)"); + printf("Message type: DHCPINFORM (8)\n"); break; default: - logg("Message type: UNKNOWN (%u)", offer_packet->options[x]); + printf("Message type: UNKNOWN (%u)\n", offer_packet->options[x]); break; } } @@ -394,7 +414,7 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack number = ntohs(number); else if(optlen == 4) number = ntohl(number); - logg("%s: %u", opttab[i].name, number); + printf("%s: %u\n", opttab[i].name, number); } } } @@ -411,7 +431,7 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack // possible "(empty)" char buffer[4*optlen + 9]; binbuf_to_escaped_C_literal(&offer_packet->options[x], optlen, buffer, sizeof(buffer)); - logg("wpad-server: \"%s\"", buffer); + printf("wpad-server: \"%s\"\n", buffer); } else if(opttype == 158) // DHCPv4 PCP Option (RFC 7291) { // https://tools.ietf.org/html/rfc7291#section-4 @@ -424,9 +444,9 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack break; memcpy(&addr_list.s_addr, &offer_packet->options[x+n*sizeof(addr_list.s_addr)], sizeof(addr_list.s_addr)); if(n > 0) - logg_sameline(" "); + printf(" "); - logg("Port Control Protocol (PCP) server: %s", inet_ntoa(addr_list)); + printf("Port Control Protocol (PCP) server: %s\n", inet_ntoa(addr_list)); } } else if((opttype == 121 || opttype == 249) && optlen > 4) @@ -435,7 +455,7 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack // see // - https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-dhcpe/f9c19c79-1c7f-4746-b555-0c0fc523f3f9 // - https://datatracker.ietf.org/doc/html/rfc3442 (page 3) - logg("%s Classless Static Route:", opttype == 121 ? "RFC 3442" : "Microsoft"); + printf("%s Classless Static Route:", opttype == 121 ? "RFC 3442" : "Microsoft"); // Loop over contained routes unsigned int n = 0; for(unsigned int i = 1; n < optlen; i++) @@ -460,26 +480,26 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack if(cidr == 0) { // default route (0.0.0.0/0) - logg(" %d: default via %d.%d.%d.%d", i, - router[0], router[1], router[2], router[3]); + printf(" %d: default via %d.%d.%d.%d", i, + router[0], router[1], router[2], router[3]); } else { // specific route - logg(" %d: %d.%d.%d.%d/%d via %d.%d.%d.%d", i, - addr[0], addr[1], addr[2], addr[3], cidr, - router[0], router[1], router[2], router[3]); + printf(" %d: %d.%d.%d.%d/%d via %d.%d.%d.%d", i, + addr[0], addr[1], addr[2], addr[3], cidr, + router[0], router[1], router[2], router[3]); } } } else { - logg_sameline("Unknown option %d:", opttype); + printf("Unknown option %d:", opttype); // Print bytes for(unsigned i = 0; i < optlen; i++) - logg_sameline(" %02X", (unsigned char)offer_packet->options[x+i]); + printf(" %02X", (unsigned char)offer_packet->options[x+i]); // Add newline when done above - logg(" (length %d)", optlen); + printf(" (length %d)\n", optlen); } } @@ -488,7 +508,7 @@ static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_pack } // Add one empty line for readability - logg(" "); + printf("\n"); } // receives a DHCP packet @@ -514,13 +534,14 @@ static bool receive_dhcp_packet(void *buffer, int buffer_size, const char *iface address_size = sizeof(struct sockaddr_in); recv_result = recvfrom(sock, (char *)buffer, buffer_size, 0, (struct sockaddr *)address, &address_size); - logg("* Received %d bytes from %s:%s", recv_result, iface, inet_ntoa(address->sin_addr)); + printf_locked("\n* Received %d bytes from %s:%s\n", recv_result, iface, inet_ntoa(address->sin_addr)); #ifdef DEBUG - logg(" after waiting for %f seconds", difftime(time(NULL), start_time)); + printf_locked(" after waiting for %f seconds\n", difftime(time(NULL), start_time)); #endif // Return on error - if(recv_result == -1){ - logg(" recvfrom() failed on %s, error: %s", iface, strerror(errno)); + if(recv_result == -1) + { + printf_locked(" recvfrom() failed on %s, error: %s\n", iface, strerror(errno)); return false; } @@ -558,14 +579,14 @@ static bool get_dhcp_offer(const int sock, const uint32_t xid, const char *iface return false; #ifdef DEBUG - logg(" DHCPOFFER XID: %lu (0x%X)", (unsigned long) ntohl(offer_packet.xid), ntohl(offer_packet.xid)); + printf(" DHCPOFFER XID: %lu (0x%X)\n", (unsigned long) ntohl(offer_packet.xid), ntohl(offer_packet.xid)); #endif // check packet xid to see if its the same as the one we used in the discover packet if(ntohl(offer_packet.xid) != xid) { - logg(" DHCPOFFER XID (%lu) does not match our DHCPDISCOVER XID (%lu) - ignoring packet (not for us)\n", - (unsigned long) ntohl(offer_packet.xid), (unsigned long) xid); + printf(" DHCPOFFER XID (%lu) does not match our DHCPDISCOVER XID (%lu) - ignoring packet (not for us)\n", + (unsigned long) ntohl(offer_packet.xid), (unsigned long) xid); pthread_mutex_unlock(&lock); continue; @@ -574,73 +595,73 @@ static bool get_dhcp_offer(const int sock, const uint32_t xid, const char *iface // check hardware address if(memcmp(offer_packet.chaddr, mac, 6) != 0) { - logg(" DHCPOFFER hardware address did not match our own - ignoring packet (not for us)"); + printf(" DHCPOFFER hardware address did not match our own - ignoring packet (not for us)\n"); - logg_sameline(" DHCPREQUEST chaddr: "); + printf(" DHCPREQUEST chaddr: "); for(uint8_t x = 0; x < 6; x++) - logg_sameline("%02x%s", mac[x], x < 5 ? ":" : ""); - logg(" (our MAC address)"); + printf("%02x%s", mac[x], x < 5 ? ":" : ""); + printf(" (our MAC address)\n"); - logg_sameline(" DHCPOFFER chaddr: "); + printf(" DHCPOFFER chaddr: "); for(uint8_t x = 0; x < 6; x++) - logg_sameline("%02x%s", offer_packet.chaddr[x], x < 5 ? ":" : ""); - logg(" (response MAC address)"); + printf("%02x%s", offer_packet.chaddr[x], x < 5 ? ":" : ""); + printf(" (response MAC address)\n"); pthread_mutex_unlock(&lock); continue; } - logg_sameline(" Offered IP address: "); + printf(" Offered IP address: "); if(offer_packet.yiaddr.s_addr != 0) - logg("%s", inet_ntoa(offer_packet.yiaddr)); + printf("%s\n", inet_ntoa(offer_packet.yiaddr)); else - logg("N/A"); + printf("N/A\n"); - logg_sameline(" Server IP address: "); + printf(" Server IP address: "); if(offer_packet.siaddr.s_addr != 0) - logg("%s", inet_ntoa(offer_packet.siaddr)); + printf("%s\n", inet_ntoa(offer_packet.siaddr)); else - logg("N/A"); + printf("N/A\n"); - logg_sameline(" Relay-agent IP address: "); + printf(" Relay-agent IP address: "); if(offer_packet.giaddr.s_addr != 0) - logg("%s", inet_ntoa(offer_packet.giaddr)); + printf("%s\n", inet_ntoa(offer_packet.giaddr)); else - logg("N/A"); + printf("N/A\n"); - logg_sameline(" BOOTP server: "); + printf(" BOOTP server: "); if(offer_packet.sname[0] != 0) { size_t len = strlen(offer_packet.sname); char buffer[4*len + 9]; binbuf_to_escaped_C_literal(offer_packet.sname, len, buffer, sizeof(buffer)); - logg("%s", buffer); + printf("%s\n", buffer); } else - logg("(empty)"); + printf("(empty)\n"); - logg_sameline(" BOOTP file: "); + printf(" BOOTP file: "); if(offer_packet.file[0] != 0) { size_t len = strlen(offer_packet.file); char buffer[4*len + 9]; binbuf_to_escaped_C_literal(offer_packet.file, len, buffer, sizeof(buffer)); - logg("%s", buffer); + printf("%s\n", buffer); } else - logg("(empty)"); + printf("(empty)\n"); - logg(" DHCP options:"); + printf(" DHCP options:\n"); print_dhcp_offer(source.sin_addr, &offer_packet); pthread_mutex_unlock(&lock); valid_responses++; } #ifdef DEBUG - logg(" Responses seen while scanning: %d", responses); - logg(" Responses meant for this machine: %d\n", valid_responses); + printf(" Responses seen while scanning: %d\n", responses); + printf(" Responses meant for this machine: %d\n\n", valid_responses); #endif - logg("DHCP packets received on interface %s: %u", iface, valid_responses); + printf("DHCP packets received on interface %s: %u\n", iface, valid_responses); return true; } @@ -673,18 +694,21 @@ static void *dhcp_discover_iface(void *args) // Send DHCPDISCOVER packet to interface address struct sockaddr_in ifaddr = { 0 }; memcpy(&ifaddr, ((struct ifaddrs*)args)->ifa_addr, sizeof(ifaddr)); - send_dhcp_discover(dhcp_socket, xid, iface, mac, ifaddr.sin_addr.s_addr); + if(!send_dhcp_discover(dhcp_socket, xid, iface, mac, ifaddr.sin_addr.s_addr)) + goto end_dhcp_discover_iface; } else { // Probe distant servers // Send DHCPDISCOVER packet to broadcast address - send_dhcp_discover(dhcp_socket, xid, iface, mac, INADDR_BROADCAST); + if(!send_dhcp_discover(dhcp_socket, xid, iface, mac, INADDR_BROADCAST)) + goto end_dhcp_discover_iface; } // wait for a DHCPOFFER packet get_dhcp_offer(dhcp_socket, xid, iface, mac); +end_dhcp_discover_iface: // close socket we created close(dhcp_socket); @@ -708,8 +732,8 @@ int run_dhcp_discover(void) // Only print to terminal, disable log file log_ctrl(false, true); - logg("Scanning all your interfaces for DHCP servers"); - logg("Timeout: %d seconds\n", DHCPOFFER_TIMEOUT); + printf("Scanning all your interfaces for DHCP servers\n"); + printf("Timeout: %d seconds\n", DHCPOFFER_TIMEOUT); // Get interface names for available interfaces on this machine // and launch a thread for each one @@ -741,8 +765,8 @@ int run_dhcp_discover(void) { if(pthread_create(&scanthread[tid], &attr, dhcp_discover_iface, tmp ) != 0) { - logg("Unable to launch thread for interface %s, skipping...", - tmp->ifa_name); + printf_locked("Unable to launch thread for interface %s, skipping...", + tmp->ifa_name); continue; } From 25a1d0683497d675d20be81f1fe57264c248a813 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 20 May 2023 22:20:42 +0200 Subject: [PATCH 63/78] Query IPv4-capable interfaces instead of packet-interfaces when scanning for DHCP servers Signed-off-by: DL6ER --- src/tools/dhcp-discover.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 18b415b3b..91a6189f5 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -760,8 +760,8 @@ int run_dhcp_discover(void) int tid = 0; while(tmp != NULL && tid < MAXTHREADS) { - // Create a thread for interfaces of type AF_PACKET - if(tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_PACKET) + // Create a thread for interfaces of type AF_INET (IPv4) + if(tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_INET) { if(pthread_create(&scanthread[tid], &attr, dhcp_discover_iface, tmp ) != 0) { From 6b531e7c274c4979bddf55dca0fef49cda3d9bda Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 21 May 2023 09:32:41 +0200 Subject: [PATCH 64/78] Improve deplay.sh script to check against exact matches instead of regex-matching the searched string against the entire collapsed array to avoid incorrect partial matches Signed-off-by: DL6ER --- deploy.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/deploy.sh b/deploy.sh index fb7c3a4bb..832502367 100755 --- a/deploy.sh +++ b/deploy.sh @@ -42,11 +42,18 @@ for dir in "${path[@]}"; do ls -1" )" - # Only try to create the subdir if does not already exist - if [[ "${dir_content[*]}" =~ "${dir}" ]]; then - echo "Dir: ${old_path}/${dir} already exists" - else - echo "Creating dir: ${old_path}/${dir}" + # Loop over the dir content and check if this exact dir already exists + path_exists=0 + for content in "${dir_content[@]}"; do + if [[ "${content}" == "${dir}" ]]; then + echo "Dir: ${old_path}/${dir} already exists" + path_exists=1 + fi + done + + # If the dir does not exist, create it + if [[ "${path_exists}" -eq 0 ]]; then + echo "Dir: ${old_path}/${dir} does not exist. Creating it." sftp -b - "${USER}"@"${HOST}" <<< "cd ${old_path} -mkdir ${dir}" fi From 1a02c102fca3de982a4d51fd6263af8b4975ed38 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 21 May 2023 10:06:08 +0200 Subject: [PATCH 65/78] Improve message when packet is rejected by wireguard interfaces Signed-off-by: DL6ER --- src/tools/dhcp-discover.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 91a6189f5..86bf460d1 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -247,10 +247,11 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i { // strerror() returns "Required key not available" for ENOKEY // which is not helpful at all so we substitute a more - // meaningful error message for ENOKEY, which is returned when - // the network is unreachable - const char *error = errno == ENOKEY ? "Broadcast is unreachable" : strerror(errno); - printf_locked("Error: Could not send DHCPDISCOVER packet on interface %s: %s\n", iface, error); + // meaningful error message for ENOKEY returned by wireguard interfaces + // (see https://www.wireguard.com/papers/wireguard.pdf, page 5) + const char *error = errno == ENOKEY ? "No such peer" : strerror(errno); + printf_locked("Error: Could not send DHCPDISCOVER packet to %s on interface %s: %s\n", + inet_ntoa(target.sin_addr), iface, error); return false; } From d75305b1ed8dc00827618c921b8702d0c34fa6eb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 21 May 2023 12:57:41 +0200 Subject: [PATCH 66/78] Ensure we are in lock-mode when printing the final result Signed-off-by: DL6ER --- src/tools/dhcp-discover.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 86bf460d1..ea7915641 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -249,7 +249,7 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i // which is not helpful at all so we substitute a more // meaningful error message for ENOKEY returned by wireguard interfaces // (see https://www.wireguard.com/papers/wireguard.pdf, page 5) - const char *error = errno == ENOKEY ? "No such peer" : strerror(errno); + const char *error = errno == ENOKEY ? "No route to host (no such peer available)" : strerror(errno); printf_locked("Error: Could not send DHCPDISCOVER packet to %s on interface %s: %s\n", inet_ntoa(target.sin_addr), iface, error); return false; @@ -550,7 +550,7 @@ static bool receive_dhcp_packet(void *buffer, int buffer_size, const char *iface } // waits for a DHCPOFFER message from one or more DHCP servers -static bool get_dhcp_offer(const int sock, const uint32_t xid, const char *iface, unsigned char *mac) +static int get_dhcp_offer(const int sock, const uint32_t xid, const char *iface, unsigned char *mac) { dhcp_packet_data offer_packet; struct sockaddr_in source; @@ -577,7 +577,7 @@ static bool get_dhcp_offer(const int sock, const uint32_t xid, const char *iface #endif if(pthread_mutex_lock(&lock) != 0) - return false; + return -1; #ifdef DEBUG printf(" DHCPOFFER XID: %lu (0x%X)\n", (unsigned long) ntohl(offer_packet.xid), ntohl(offer_packet.xid)); @@ -662,12 +662,12 @@ static bool get_dhcp_offer(const int sock, const uint32_t xid, const char *iface printf(" Responses seen while scanning: %d\n", responses); printf(" Responses meant for this machine: %d\n\n", valid_responses); #endif - printf("DHCP packets received on interface %s: %u\n", iface, valid_responses); - return true; + return valid_responses; } static void *dhcp_discover_iface(void *args) { + int valid_responses = -1; // Get interface details const char *iface = ((struct ifaddrs*)args)->ifa_name; @@ -679,7 +679,7 @@ static void *dhcp_discover_iface(void *args) // Cannot create socket, likely a permission error if(dhcp_socket < 0) - pthread_exit(NULL); + goto end_dhcp_discover_iface; // get hardware address of client machine unsigned char mac[MAX_DHCP_CHADDR_LENGTH] = { 0 }; @@ -707,11 +707,15 @@ static void *dhcp_discover_iface(void *args) } // wait for a DHCPOFFER packet - get_dhcp_offer(dhcp_socket, xid, iface, mac); + valid_responses = get_dhcp_offer(dhcp_socket, xid, iface, mac); end_dhcp_discover_iface: - // close socket we created - close(dhcp_socket); + // Close socket if we created one + if(dhcp_socket > 0) + close(dhcp_socket); + + if(valid_responses >= 0) + printf_locked("DHCP packets received on interface %s: %u\n", iface, valid_responses); pthread_exit(NULL); } From b38ed4057a7935adc36353160c665f79ff5d68e3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 21 May 2023 13:44:37 +0200 Subject: [PATCH 67/78] Skip interfaces that are either down or are of loopback type Signed-off-by: DL6ER --- src/tools/dhcp-discover.c | 123 ++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 64 deletions(-) diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index ea7915641..ebdefc627 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -148,7 +148,7 @@ int get_hardware_address(const int sock, const char *iname, unsigned char *mac) int ret = 0; if((ret = ioctl(sock, SIOCGIFHWADDR, &ifr)) < 0) { - printf_locked(" Error: Could not get hardware address of interface '%s' (socket %d, error: %s)\n", iname, sock, strerror(errno)); + printf_locked(" Error: Could not get hardware address of interface %s: %s\n", iname, strerror(errno)); return false; } memcpy(&mac[0], &ifr.ifr_hwaddr.sa_data, 6); @@ -161,80 +161,72 @@ int get_hardware_address(const int sock, const char *iname, unsigned char *mac) return true; } -typedef struct dhcp_packet_struct +struct dhcp_packet_data { - u_int8_t op; // packet type - u_int8_t htype; // type of hardware address for this machine (Ethernet, etc) - u_int8_t hlen; // length of hardware address (of this machine) - u_int8_t hops; // hops - u_int32_t xid; // random transaction id number - chosen by this machine - u_int16_t secs; // seconds used in timing - u_int16_t flags; // flags - struct in_addr ciaddr; // IP address of this machine (if we already have one) - struct in_addr yiaddr; // IP address of this machine (offered by the DHCP server) - struct in_addr siaddr; // IP address of DHCP server - struct in_addr giaddr; // IP address of DHCP relay - unsigned char chaddr [MAX_DHCP_CHADDR_LENGTH]; // hardware address of this machine - char sname [MAX_DHCP_SNAME_LENGTH]; // name of DHCP server - char file [MAX_DHCP_FILE_LENGTH]; // boot file name (used for diskless booting?) - char options[MAX_DHCP_OPTIONS_LENGTH]; // options -} dhcp_packet_data; - -#define BOOTREQUEST 1 -#define BOOTREPLY 2 + u_int8_t op; // packet type + u_int8_t htype; // type of hardware address for this machine (Ethernet, etc) + u_int8_t hlen; // length of hardware address (of this machine) + u_int8_t hops; // hops + u_int32_t xid; // random transaction id number - chosen by this machine + u_int16_t secs; // seconds used in timing + u_int16_t flags; // flags + struct in_addr ciaddr; // IP address of this machine (if we already have one) + struct in_addr yiaddr; // IP address of this machine (offered by the DHCP server) + struct in_addr siaddr; // IP address of DHCP server + struct in_addr giaddr; // IP address of DHCP relay + unsigned char chaddr [MAX_DHCP_CHADDR_LENGTH]; // hardware address of this machine + char sname [MAX_DHCP_SNAME_LENGTH]; // name of DHCP server + char file [MAX_DHCP_FILE_LENGTH]; // boot file name (used for diskless booting?) + char options[MAX_DHCP_OPTIONS_LENGTH]; // options +}; // sends a DHCPDISCOVER message to the specified in an attempt to find DHCP servers -static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *iface, unsigned char *mac, const in_addr_t addr) +static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *iface, unsigned char *mac) { - dhcp_packet_data discover_packet; + struct dhcp_packet_data discover_packet = { 0 }; - // clear the packet data structure - memset(&discover_packet, 0, sizeof(discover_packet)); + // Boot request flag (backward compatible with BOOTP servers) + discover_packet.op = 1; // BOOTREQUEST - // boot request flag (backward compatible with BOOTP servers) - discover_packet.op = BOOTREQUEST; + // Hardware address type + discover_packet.htype = 1; // ETHERNET_HARDWARE_ADDRESS - // hardware address type - discover_packet.htype = 1; // ETHERNET_HARDWARE_ADDRESS; - - // length of our hardware address - discover_packet.hlen = 6; // ETHERNET_HARDWARE_ADDRESS_LENGTH; + // Length of our hardware address + discover_packet.hlen = 6; // ETHERNET_HARDWARE_ADDRESS_LENGTH discover_packet.hops = 0; - // transaction id is supposed to be random + // Transaction id is supposed to be random discover_packet.xid = htonl(xid); - ntohl(discover_packet.xid); discover_packet.secs = 0x00; - // tell server it should broadcast its response + // Tell server it should broadcast its response discover_packet.flags = htons(32768); // DHCP_BROADCAST_FLAG - // our hardware address + // Our hardware address memcpy(discover_packet.chaddr, mac, 6); - // first four bytes of options field is magic cookie (as per RFC 2132) + // First four bytes of options field are the magic cookie (as per RFC 2132) discover_packet.options[0] = '\x63'; discover_packet.options[1] = '\x82'; discover_packet.options[2] = '\x53'; discover_packet.options[3] = '\x63'; // DHCP message type is embedded in options field - discover_packet.options[4] = 53; // DHCP message type option identifier - discover_packet.options[5] = '\x01'; // DHCP message option length in bytes - discover_packet.options[6] = 1; // DHCP message type code for DHCPDISCOVER + discover_packet.options[4] = 53; // DHCP message type option identifier + discover_packet.options[5] = 1; // DHCP message option length in bytes + discover_packet.options[6] = 1; // DHCP message type code for DHCPDISCOVER // Place end option at the end of the options discover_packet.options[7] = 255; - // send the DHCPDISCOVER packet to the specified address - struct sockaddr_in target; + // Send the DHCPDISCOVER packet to the specified address + struct sockaddr_in target = { 0 }; target.sin_family = AF_INET; target.sin_port = htons(DHCP_SERVER_PORT); - target.sin_addr.s_addr = addr; - memset(&target.sin_zero, 0, sizeof(target.sin_zero)); + target.sin_addr.s_addr = INADDR_BROADCAST; #ifdef DEBUG - printf_locked("Sending DHCPDISCOVER on interface %s:%s ... \n", iface, inet_ntoa(target.sin_addr)); + printf_locked("Sending DHCPDISCOVER on interface %s@%s ... \n", inet_ntoa(target.sin_addr), iface); printf_locked("DHCPDISCOVER XID: %lu (0x%X)\n", (unsigned long) ntohl(discover_packet.xid), ntohl(discover_packet.xid)); printf_locked("DHCDISCOVER ciaddr: %s\n", inet_ntoa(discover_packet.ciaddr)); printf_locked("DHCDISCOVER yiaddr: %s\n", inet_ntoa(discover_packet.yiaddr)); @@ -250,7 +242,7 @@ static bool send_dhcp_discover(const int sock, const uint32_t xid, const char *i // meaningful error message for ENOKEY returned by wireguard interfaces // (see https://www.wireguard.com/papers/wireguard.pdf, page 5) const char *error = errno == ENOKEY ? "No route to host (no such peer available)" : strerror(errno); - printf_locked("Error: Could not send DHCPDISCOVER packet to %s on interface %s: %s\n", + printf_locked("Error: Could not send DHCPDISCOVER to %s@%s: %s\n", inet_ntoa(target.sin_addr), iface, error); return false; } @@ -274,7 +266,7 @@ static void gen_249_test_data(dhcp_packet_data *offer_packet) #endif // adds a DHCP OFFER to list in memory -static void print_dhcp_offer(struct in_addr source, dhcp_packet_data *offer_packet) +static void print_dhcp_offer(struct in_addr source, struct dhcp_packet_data *offer_packet) { if(offer_packet == NULL) return; @@ -552,7 +544,7 @@ static bool receive_dhcp_packet(void *buffer, int buffer_size, const char *iface // waits for a DHCPOFFER message from one or more DHCP servers static int get_dhcp_offer(const int sock, const uint32_t xid, const char *iface, unsigned char *mac) { - dhcp_packet_data offer_packet; + struct dhcp_packet_data offer_packet; struct sockaddr_in source; #ifdef DEBUG unsigned int responses = 0; @@ -689,22 +681,9 @@ static void *dhcp_discover_iface(void *args) srand(time(NULL)); const uint32_t xid = random(); - if(strcmp(iface, "lo") == 0) - { - // Probe a local server listening on this interface - // Send DHCPDISCOVER packet to interface address - struct sockaddr_in ifaddr = { 0 }; - memcpy(&ifaddr, ((struct ifaddrs*)args)->ifa_addr, sizeof(ifaddr)); - if(!send_dhcp_discover(dhcp_socket, xid, iface, mac, ifaddr.sin_addr.s_addr)) - goto end_dhcp_discover_iface; - } - else - { - // Probe distant servers - // Send DHCPDISCOVER packet to broadcast address - if(!send_dhcp_discover(dhcp_socket, xid, iface, mac, INADDR_BROADCAST)) - goto end_dhcp_discover_iface; - } + // Probe servers on this interface + if(!send_dhcp_discover(dhcp_socket, xid, iface, mac)) + goto end_dhcp_discover_iface; // wait for a DHCPOFFER packet valid_responses = get_dhcp_offer(dhcp_socket, xid, iface, mac); @@ -768,10 +747,26 @@ int run_dhcp_discover(void) // Create a thread for interfaces of type AF_INET (IPv4) if(tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_INET) { + // Skip if interface is down + if(!(tmp->ifa_flags & IFF_UP)) + { + tmp = tmp->ifa_next; + continue; + } + + // Skip if interface is loopback + if(tmp->ifa_flags & IFF_LOOPBACK) + { + tmp = tmp->ifa_next; + continue; + } + + // Create a probing thread for this interface if(pthread_create(&scanthread[tid], &attr, dhcp_discover_iface, tmp ) != 0) { printf_locked("Unable to launch thread for interface %s, skipping...", tmp->ifa_name); + tmp = tmp->ifa_next; continue; } From 59a11808a609e63e1aaabcb3342f9c32d2badf5d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 21 May 2023 17:39:17 +0200 Subject: [PATCH 68/78] Do not try to scan for DHCP servers in network where the kernel knows that there is no broadcasting support (e.g. Wireguard) Signed-off-by: DL6ER --- src/tools/dhcp-discover.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index ebdefc627..29738894e 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -747,15 +747,13 @@ int run_dhcp_discover(void) // Create a thread for interfaces of type AF_INET (IPv4) if(tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_INET) { - // Skip if interface is down - if(!(tmp->ifa_flags & IFF_UP)) - { - tmp = tmp->ifa_next; - continue; - } - - // Skip if interface is loopback - if(tmp->ifa_flags & IFF_LOOPBACK) + // Skip if ... + // - interface is not up + // - broadcast is not supported + // - interface is loopback net + if(!(tmp->ifa_flags & IFF_UP) || + !(tmp->ifa_flags & IFF_BROADCAST) || + tmp->ifa_flags & IFF_LOOPBACK) { tmp = tmp->ifa_next; continue; From 88502af00bdfe4cb4ba8710c47370fe53dae4b57 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 21 May 2023 17:43:25 +0200 Subject: [PATCH 69/78] Do not run ARP scans in networks where the kernel knows that ARP is not supported (e.g. Wireguard) Signed-off-by: DL6ER --- src/tools/arp-scan.c | 12 ++++++++++++ src/tools/dhcp-discover.c | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/tools/arp-scan.c b/src/tools/arp-scan.c index 46e3496f3..84030a7cf 100644 --- a/src/tools/arp-scan.c +++ b/src/tools/arp-scan.c @@ -632,6 +632,18 @@ int run_arp_scan(const bool scan_all, const bool extreme_mode) // Create a thread for interfaces of type AF_INET if(tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_INET) { + // Skip interface scan if ... + // - interface is not up + // - ARP is not supported + // - interface is loopback net + if(!(tmp->ifa_flags & IFF_UP) || + (tmp->ifa_flags & IFF_NOARP) || + (tmp->ifa_flags & IFF_LOOPBACK)) + { + tmp = tmp->ifa_next; + continue; + } + thread_data[tid].ifa = tmp; strncpy(thread_data[tid].iface, tmp->ifa_name, sizeof(thread_data[tid].iface) - 1); diff --git a/src/tools/dhcp-discover.c b/src/tools/dhcp-discover.c index 29738894e..b9d920cb6 100644 --- a/src/tools/dhcp-discover.c +++ b/src/tools/dhcp-discover.c @@ -747,7 +747,7 @@ int run_dhcp_discover(void) // Create a thread for interfaces of type AF_INET (IPv4) if(tmp->ifa_addr && tmp->ifa_addr->sa_family == AF_INET) { - // Skip if ... + // Skip interface scan if ... // - interface is not up // - broadcast is not supported // - interface is loopback net From 9689b9856fb4667ff47fe7f5543f7781d2dc74cc Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Mon, 22 May 2023 17:28:30 -0300 Subject: [PATCH 70/78] Allowing underscore and hyfen in any position for gravity parseList Signed-off-by: RD WebDesign --- src/gravity-tools.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index 523fec54a..b53a57c07 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -18,7 +18,7 @@ // Adapted from https://stackoverflow.com/a/30007882 // - Added "(?:...)" to form non-capturing groups (slightly faster) #define TLD_PATTERN "[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" -#define SUBDOMAIN_PATTERN "([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\\.)" +#define SUBDOMAIN_PATTERN "([a-z0-9_-]{0,63}\\.)" // supported exact style: subdomain.domain.tld // SUBDOMAIN_PATTERN is mandatory for exact style, disallowing TLD blocking From 2a13beb3c26d07bffb231a5cdcb8539f84770e0a Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Tue, 23 May 2023 17:01:45 -0300 Subject: [PATCH 71/78] Do not consider false positives as invalid domains Signed-off-by: RD WebDesign --- src/gravity-tools.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index b53a57c07..f5b6db101 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -247,26 +247,30 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis } else { - // No match - add to list of invalid domains - if(invalid_domains_list_len < MAX_INVALID_DOMAINS) + // Ignore false positives - they don't count as invalid domains + if(regexec(&false_positives_regex, line, 0, NULL, 0) != 0) { - // Check if we have this domain already - bool found = false; - for(unsigned int i = 0; i < invalid_domains_list_len; i++) + // No match - add to list of invalid domains + if(invalid_domains_list_len < MAX_INVALID_DOMAINS) { - if(strcmp(invalid_domains_list[i], line) == 0) + // Check if we have this domain already + bool found = false; + for(unsigned int i = 0; i < invalid_domains_list_len; i++) { - found = true; - break; + if(strcmp(invalid_domains_list[i], line) == 0) + { + found = true; + break; + } } - } - // If not found, check if this is a false - // positive and add it to the list if it is not - if(!found && regexec(&false_positives_regex, line, 0, NULL, 0) != 0) - invalid_domains_list[invalid_domains_list_len++] = strdup(line); + // If not found, add it to the list + if(!found) + invalid_domains_list[invalid_domains_list_len++] = strdup(line); + + } + invalid_domains++; } - invalid_domains++; } // Print progress if the file is large enough every 100 lines From f4374876df1823f94ede0ba8135a701d49d9fe96 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 24 May 2023 15:55:43 -0300 Subject: [PATCH 72/78] Improving the comments Signed-off-by: RD WebDesign --- src/gravity-tools.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index f5b6db101..6cfa9b1d9 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -247,10 +247,13 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis } else { + // No match - This is an invalid domain or a false positive + // Ignore false positives - they don't count as invalid domains if(regexec(&false_positives_regex, line, 0, NULL, 0) != 0) { - // No match - add to list of invalid domains + // Add the domain to invalid_domains_list only + // if the list contains < MAX_INVALID_DOMAINS if(invalid_domains_list_len < MAX_INVALID_DOMAINS) { // Check if we have this domain already From 825146fa5957c8976f4188534021aad99bec4e2a Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 24 May 2023 16:00:44 -0300 Subject: [PATCH 73/78] Adding anchors to false_positives_regex Signed-off-by: RD WebDesign --- src/gravity-tools.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index 6cfa9b1d9..852a2bb5c 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -35,7 +35,7 @@ // A list of items of common local hostnames not to report as unusable // Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files // but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output -#define FALSE_POSITIVES "localhost|localhost.localdomain|local|broadcasthost|localhost|ip6-localhost|ip6-loopback|lo0 localhost|ip6-localnet|ip6-mcastprefix|ip6-allnodes|ip6-allrouters|ip6-allhosts" +#define FALSE_POSITIVES "^(localhost|localhost.localdomain|local|broadcasthost|localhost|ip6-localhost|ip6-loopback|lo0 localhost|ip6-localnet|ip6-mcastprefix|ip6-allnodes|ip6-allrouters|ip6-allhosts)$" // Print progress for files larger than 10 MB // This is to avoid printing progress for small files From 14dc1311bcc1e43e626813aa69d30850da8320ee Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 May 2023 17:00:26 +0200 Subject: [PATCH 74/78] Update adlist.date_updated in parseList command Signed-off-by: DL6ER --- src/gravity-tools.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gravity-tools.c b/src/gravity-tools.c index 852a2bb5c..ddf638f91 100644 --- a/src/gravity-tools.c +++ b/src/gravity-tools.c @@ -317,7 +317,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis } // Update number of domains on this list - sql = "UPDATE adlist SET number = ?, invalid_domains = ? WHERE id = ?;"; + sql = "UPDATE adlist SET number = ?, invalid_domains = ?, date_updated = cast(strftime('%s', 'now') as int) WHERE id = ?;"; if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { printf("%s %s Unable to prepare SQL statement to update adlist properties in database file %s\n", @@ -406,4 +406,4 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis // Return success return EXIT_SUCCESS; -} \ No newline at end of file +} From 15faf3ca001284b78462cc27ad34b6854034d8ec Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 May 2023 17:02:55 +0200 Subject: [PATCH 75/78] Rename src/{gravity-tools.* => tools/gravity-parseList.*} Signed-off-by: DL6ER --- src/CMakeLists.txt | 2 -- src/args.c | 2 +- src/tools/CMakeLists.txt | 2 ++ src/{gravity-tools.c => tools/gravity-parseList.c} | 4 ++-- src/{gravity-tools.h => tools/gravity-parseList.h} | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) rename src/{gravity-tools.c => tools/gravity-parseList.c} (99%) rename src/{gravity-tools.h => tools/gravity-parseList.h} (91%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0b7c4ec6e..cee7b1b32 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -128,8 +128,6 @@ set(sources FTL.h gc.c gc.h - gravity-tools.c - gravity-tools.h log.c log.h main.c diff --git a/src/args.c b/src/args.c index f7e1c2e93..fc49a68c2 100644 --- a/src/args.c +++ b/src/args.c @@ -33,7 +33,7 @@ // LUA dependencies #include "lua/ftl_lua.h" // gravity_parseList() -#include "gravity-tools.h" +#include "tools/gravity-parseList.h" // run_dhcp_discover() #include "tools/dhcp-discover.h" // run_arp_scan() diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 47de88b1a..47b9e5e10 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -13,6 +13,8 @@ set(tools_sources arp-scan.h dhcp-discover.c dhcp-discover.h + gravity-parseList.c + gravity-parseList.h ) add_library(tools OBJECT ${tools_sources}) diff --git a/src/gravity-tools.c b/src/tools/gravity-parseList.c similarity index 99% rename from src/gravity-tools.c rename to src/tools/gravity-parseList.c index ddf638f91..8fb7d1fac 100644 --- a/src/gravity-tools.c +++ b/src/tools/gravity-parseList.c @@ -3,12 +3,12 @@ * Network-wide ad blocking via your own hardware. * * FTL Engine -* Gravity tools collection routines +* Gravity parseList routines * * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ -#include "gravity-tools.h" +#include "tools/gravity-parseList.h" #include "args.h" #include #include "database/sqlite3.h" diff --git a/src/gravity-tools.h b/src/tools/gravity-parseList.h similarity index 91% rename from src/gravity-tools.h rename to src/tools/gravity-parseList.h index 3cc021711..fb09c1d2e 100644 --- a/src/gravity-tools.h +++ b/src/tools/gravity-parseList.h @@ -3,7 +3,7 @@ * Network-wide ad blocking via your own hardware. * * FTL Engine -* Gravity tools collection prototypes +* Gravity parseList prototypes * * This file is copyright under the latest version of the EUPL. * Please see LICENSE file for your rights under this license. */ From 42b36cf5d1038d1f77ee4968242b1ba3d5dc8efd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 May 2023 21:50:23 +0200 Subject: [PATCH 76/78] Update src/tools/gravity-parseList.c Co-authored-by: yubiuser Signed-off-by: DL6ER --- src/tools/gravity-parseList.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index 8fb7d1fac..f49bcb7b2 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -316,7 +316,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis } } - // Update number of domains on this list + // Update number of domains and update timestamp on this list sql = "UPDATE adlist SET number = ?, invalid_domains = ?, date_updated = cast(strftime('%s', 'now') as int) WHERE id = ?;"; if(sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { From 487d4d2768e72839581bc137c76a70e6df70ecc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 27 May 2023 22:19:05 +0200 Subject: [PATCH 77/78] Remove code duplication found in gravit.sh gravity_ParseFileIntoDomains() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/tools/gravity-parseList.c | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index f49bcb7b2..450d37b04 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -157,34 +157,14 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis total_read += read; lineno++; - // Remove leading whitespace - // isspace() matches spaces, tabs, form feeds, line breaks, - // carriage returns, and vertical tabs - while(isspace(line[0])) - line++; - - // Skip comments - // # is used for comments in HOSTS files - // ! is used for comments in AdBlock-style files - // [ is used for comments in AdGuard-style files and ABP headers - if(line[0] == '#' || line[0] == '!' || line[0] == '[') - continue; - // Remove trailing newline if(line[read-1] == '\n') line[--read] = '\0'; - // Remove trailing carriage return (Windows) - if(line[read-1] == '\r') - line[--read] = '\0'; - // Remove trailing dot (convert FQDN to domain) if(line[read-1] == '.') line[--read] = '\0'; - // Skip empty lines - if(line[0] == '\0') - continue; regmatch_t match = { 0 }; // Validate line @@ -219,13 +199,6 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis { // ABP-style match (see comments above) - // Skip lines containing ABP extended CSS selectors - // ("##", "#!#", "#@#", "#?#") preceded by a letter - // See https://github.com/pi-hole/pi-hole/pull/5247 for - // further information on why this is necessary - if(regexec(&abp_css_regex, line, 1, &match, 0) == 0) - continue; - // Append pattern to database using prepared statement if(sqlite3_bind_text(stmt, 1, line, -1, SQLITE_STATIC) != SQLITE_OK) { From dad2b9f8927cceebe14763f14a4ac871f4fc4cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 27 May 2023 22:46:01 +0200 Subject: [PATCH 78/78] Remove traces of ABP_CSS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- src/tools/gravity-parseList.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/tools/gravity-parseList.c b/src/tools/gravity-parseList.c index 450d37b04..3964cf569 100644 --- a/src/tools/gravity-parseList.c +++ b/src/tools/gravity-parseList.c @@ -29,9 +29,6 @@ // See https://github.com/pi-hole/pi-hole/pull/5240 #define ABP_DOMAIN_REXEX "\\|\\|"SUBDOMAIN_PATTERN"*"TLD_PATTERN"\\^" -// Detects ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter -#define ABP_CSS_SELECTORS "[a-z]#[$?@]{0,1}#" - // A list of items of common local hostnames not to report as unusable // Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files // but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output @@ -75,7 +72,7 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis rewind(fpin); // Compile regular expression to validate domains - regex_t exact_regex, abp_regex, false_positives_regex, abp_css_regex; + regex_t exact_regex, abp_regex, false_positives_regex; if(regcomp(&exact_regex, VALID_DOMAIN_REXEX, REG_EXTENDED) != 0) { printf("%s %s Unable to compile regular expression to validate exact domains\n", @@ -100,14 +97,6 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis sqlite3_close(db); return EXIT_FAILURE; } - if(regcomp(&abp_css_regex, ABP_CSS_SELECTORS, REG_EXTENDED) != 0) - { - printf("%s %s Unable to compile regular expression to validate ABP-style domains\n", - over, cross); - fclose(fpin); - sqlite3_close(db); - return EXIT_FAILURE; - } // Begin transaction if(sqlite3_exec(db, "BEGIN TRANSACTION;", NULL, NULL, NULL) != SQLITE_OK) @@ -368,7 +357,6 @@ int gravity_parseList(const char *infile, const char *outfile, const char *adlis regfree(&exact_regex); regfree(&abp_regex); regfree(&false_positives_regex); - regfree(&abp_css_regex); for(unsigned int i = 0; i < invalid_domains_list_len; i++) if(invalid_domains_list[i] != NULL) free(invalid_domains_list[i]);