Merge pull request #2369 from greenbone/speed-up-initial-scap-update

Speed up CPEs and CPE match strings update by using COPY statements

doc/gvmd.8

@@ -217,6 +217,9 @@ Time out tasks that are more than TIME minutes overdue. -1 to disable, 0 for min
 \fB--secinfo-commit-size=\fINUMBER\fB\f1
 During CERT and SCAP sync, commit updates to the database every NUMBER items, 0 for unlimited.
 .TP
+\fB--secinfo-fast_init=\fINUMBER\fB\f1
+Whether to prefer faster SQL with less checks for non-incremental SecInfo updates. 0 to use statements with more checks, 1 to use faster statements, default: 1
+.TP
 \fB-c, --unix-socket=\fIFILENAME\fB\f1
 Listen on UNIX socket at FILENAME.
 .TP

doc/gvmd.8.xml

@@ -508,6 +508,14 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
            NUMBER items, 0 for unlimited.</p>
       </optdesc>
     </option>
+    <option>
+      <p><opt>--secinfo-fast_init=<arg>NUMBER</arg></opt></p>
+      <optdesc>
+        <p>Whether to prefer faster SQL with less checks for non-incremental
+           SecInfo updates. 0 to use statements with more checks, 1 to use
+           faster statements, default: 1</p>
+      </optdesc>
+    </option>
     <option>
       <p><opt>-c, --unix-socket=<arg>FILENAME</arg></opt></p>
       <optdesc>

doc/gvmd.html

@@ -451,6 +451,14 @@ <h2>Options</h2>
 
 
 
+      <p><b>--secinfo-fast-init=<em>NUMBER</em></b></p>
+
+        <p>Whether to prefer faster SQL with less checks for non-incremental
+           SecInfo updates. 0 to use statements with more checks, 1 to use
+           faster statements, default: 1</p>
+
+
+
       <p><b>--slave-commit-size=<em>NUMBER</em></b></p>
 
         <p>During slave updates, commit after every NUMBER updated results and

src/gvmd.c

@@ -1867,6 +1867,7 @@ gvmd (int argc, char** argv, char *env[])
   static int schedule_timeout = SCHEDULE_TIMEOUT_DEFAULT;
   static int affected_products_query_size
     = AFFECTED_PRODUCTS_QUERY_SIZE_DEFAULT;
+  static int secinfo_fast_init = SECINFO_FAST_INIT_DEFAULT;
   static int secinfo_commit_size = SECINFO_COMMIT_SIZE_DEFAULT;
   static gchar *delete_scanner = NULL;
   static gchar *verify_scanner = NULL;
@@ -2242,6 +2243,13 @@ gvmd (int argc, char** argv, char *env[])
           "During CERT and SCAP sync, commit updates to the database every"
           " <number> items, 0 for unlimited, default: "
           G_STRINGIFY (SECINFO_COMMIT_SIZE_DEFAULT), "<number>" },
+        { "secinfo-fast-init", '\0', 0, G_OPTION_ARG_INT,
+          &secinfo_fast_init,
+          "Whether to prefer faster SQL with less checks for non-incremental"
+          " SecInfo updates."
+          " 0 to use statements with more checks, 1 to use faster statements,"
+          " default: "
+          G_STRINGIFY (SECINFO_FAST_INIT_DEFAULT), "<number>" },
         { "set-encryption-key", '\0', 0, G_OPTION_ARG_STRING,
           &set_encryption_key,
           "Set the encryption key with the given UID as the new default"
@@ -2370,7 +2378,9 @@ gvmd (int argc, char** argv, char *env[])
   /* Set the connection auto retry */
   set_scanner_connection_retry (scanner_connection_retry);
 
-  /* Set SQL sizes */
+  /* Set SQL sizes and related options */
+
+  set_secinfo_fast_init (secinfo_fast_init);
 
   set_affected_products_query_size (affected_products_query_size);
 

src/manage_pg.c

@@ -3848,6 +3848,40 @@ manage_db_add_constraints (const gchar *name)
   return 0;
 }
 
+/**
+ * @brief Create the indexes for the CPEs table in the scap2 schema.
+ */
+void
+create_indexes_cpe ()
+{
+  sql ("CREATE UNIQUE INDEX cpe_idx"
+       " ON scap2.cpes (name);");
+  sql ("CREATE INDEX cpes_by_creation_time_idx"
+       " ON scap2.cpes (creation_time);");
+  sql ("CREATE INDEX cpes_by_modification_time_idx"
+       " ON scap2.cpes (modification_time);");
+  sql ("CREATE INDEX cpes_by_severity"
+       " ON scap2.cpes (severity);");
+  sql ("CREATE INDEX cpes_by_uuid"
+       " ON scap2.cpes (uuid);");
+  sql ("CREATE INDEX cpes_by_cpe_name_id"
+       " ON scap2.cpes(cpe_name_id);");
+}
+
+/**
+ * @brief Remove the indexes for the CPEs table in the scap2 schema.
+ */
+void
+drop_indexes_cpe ()
+{
+  sql ("DROP INDEX IF EXISTS scap2.cpe_idx");
+  sql ("DROP INDEX IF EXISTS scap2.cpes_by_creation_time_idx");
+  sql ("DROP INDEX IF EXISTS scap2.cpes_by_modification_time_idx");
+  sql ("DROP INDEX IF EXISTS scap2.cpes_by_severity");
+  sql ("DROP INDEX IF EXISTS scap2.cpes_by_uuid");
+  sql ("DROP INDEX IF EXISTS scap2.cpes_by_cpe_name_id");
+}
+
 /**
  * @brief Init external database.
  *
@@ -3869,18 +3903,7 @@ manage_db_init_indexes (const gchar *name)
       sql ("CREATE INDEX cves_by_severity"
            " ON scap2.cves (severity);");
 
-      sql ("CREATE UNIQUE INDEX cpe_idx"
-           " ON scap2.cpes (name);");
-      sql ("CREATE INDEX cpes_by_creation_time_idx"
-           " ON scap2.cpes (creation_time);");
-      sql ("CREATE INDEX cpes_by_modification_time_idx"
-           " ON scap2.cpes (modification_time);");
-      sql ("CREATE INDEX cpes_by_severity"
-           " ON scap2.cpes (severity);");
-      sql ("CREATE INDEX cpes_by_uuid"
-           " ON scap2.cpes (uuid);");
-      sql ("CREATE INDEX cpes_by_cpe_name_id"
-           " ON scap2.cpes(cpe_name_id);");
+      create_indexes_cpe ();
 
       sql ("CREATE INDEX cpe_match_nodes_by_root_id"
            " ON scap2.cpe_match_nodes(root_id);");

src/manage_sql.h

@@ -543,4 +543,10 @@ cleanup_nvt_sequences ();
 int
 cleanup_ids_for_table (const char *);
 
+void
+create_indexes_cpe ();
+
+void
+drop_indexes_cpe ();
+
 #endif /* not _GVMD_MANAGE_SQL_H */