Develop (#227)

* Disable git error messages in log file (#179) * Remove extractGitVersion, use version.ini instead (#179) * Omit timestamp in error logger It's likely added by the operating system. * Don't show timeline of no historicalDataDir * Fix undefined variable * Start Docker image (#192) * Configuration without id is broken #194 * Configuration without id is broken #194 * Adding BnF - Bibliothèque nationale de France (French National Library) * Read schema type from the analysis output #197 * Create Hungarian translation #198: initial version and make Hungarian selectable * Create Hungarian translation #198: documentation * BaseTab: logger must be initialized first * Allow config to be a symlink * Link to field documentation in completeness tab #119 * Replace error_log with logger to support log levels #193 * Replace error_log with logger to support log levels #193: general logs * Change link to field documentation From icon link to linking the field (see #119). * Add schema type fallback * Extend docker * Update jQuery to 3.7.1 #205 * Update jQuery to 3.7.1 #205 * Update MARC schema with OGyK extension * display-xxx does not pick catalogue #209: fix issue * Shacl page: make the IDs of failed rules downloadable for #208 * Shacl page: make the IDs of failed rules downloadable for #208: download SHACL configuration and output files at download tab * display-xxx does not pick catalogue #209: fix issue * Provide Docker image #192 * Provide Docker image #192 * Negate filters #222 * Provide Docker image #192 * Quick fix for str_starts_with in PHP 7.x * Implement 'delta report' #218 --------- Co-authored-by: Jakob Voss <[email protected]>
pkiraly · Dec 1, 2024 · d5d57b7 · d5d57b7
1 parent 25a9a3c
commit d5d57b7
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 24 deletions.
diff --git a/classes/BaseTab.php b/classes/BaseTab.php
@@ -128,6 +128,10 @@ protected function getFilePath($name): string {
     return sprintf('%s/%s/%s', $this->configuration->getDir(), $this->configuration->getDirName(), $name); // ['dir']
   }
 
+  protected function getDeltaFilePath($name): string {
+    return sprintf('%s/%s/delta/%s', $this->configuration->getDir(), $this->configuration->getDirName(), $name); // ['dir']
+  }
+
   protected function getVersionedFilePath($version, $name): string {
     return sprintf('%s/_historical/%s/%s/%s', $this->configuration->getDir(), $this->configuration->getDirName(), $version, $name);
   }
@@ -577,13 +581,15 @@ protected function addParam(&$params, $object, $key, $defaulValue, $excludeKeys
       $params[] = sprintf('%s=%s', $key, urlencode($object->{$key}));
   }
 
-  protected function getDbDir() {
+  protected function getDbDir(bool $delta) {
+    if ($delta)
+      return sprintf('%s/%s/delta', $this->configuration->getDir(), $this->configuration->getDirName());
     return sprintf('%s/%s', $this->configuration->getDir(), $this->configuration->getDirName());
   }
 
-  protected function issueDB(): IssuesDB {
+  protected function issueDB(bool $delta = false): IssuesDB {
     if (is_null($this->issueDB)) {
-      $this->issueDB = new IssuesDB($this->getDbDir(), $this->log);
+      $this->issueDB = new IssuesDB($this->getDbDir($delta), $this->log);
     }
     return $this->issueDB;
   }

diff --git a/classes/Issues.php b/classes/Issues.php
@@ -19,10 +19,20 @@ class Issues extends BaseTab {
   private $listType;
   private $version;
   public $currentGroup;
+  /**
+   * @var true
+   */
+  private bool $delta = false;
   protected $parameterFile = 'validation.params.json';
 
   public function prepareData(Smarty &$smarty) {
     parent::prepareData($smarty);
+    $smarty->assign("delta", $this->delta);
+    if ($this->delta) {
+      $this->count = $this->readCount($this->getDeltaFilePath('count.csv'));
+      $smarty->assign("deltaCount", $this->count);
+    }
+
     if ($this->versioning) {
       $versions = $this->getVersions();
       $smarty->assign('versions', $versions);
@@ -52,13 +62,13 @@ public function prepareData(Smarty &$smarty) {
     if ($this->action == 'download' || $this->action == 'query') {
       $this->processDownloadOrQueryRequest();
     } elseif ($this->action == 'record') {
-        $this->processRecordRequest($smarty);
+      $this->processRecordRequest($smarty);
     } elseif ($this->action == 'ajaxIssue') {
-       $this->processAjaxIssueRequest($smarty);
+      $this->processAjaxIssueRequest($smarty);
     } elseif ($this->action == 'ajaxIssueByTag') {
-       $this->processAjaxIssueByTagRequest($smarty);
+      $this->processAjaxIssueByTagRequest($smarty);
     } else {
-       $this->processListRequest($smarty);
+      $this->processListRequest($smarty);
     }
   }
 
@@ -76,6 +86,8 @@ private function readIssues() {
     $lineNumber = 0;
     if ($this->versioning && $this->version != '') {
       $elementsFile = $this->getVersionedFilePath($this->version, 'issue-summary.csv');
+    } elseif ($this->delta) {
+      $elementsFile = $this->getDeltaFilePath('issue-summary.csv');
     } else {
       $elementsFile = $this->getFilePath('issue-summary.csv');
     }
@@ -155,11 +167,11 @@ private function readIssuesAjax($categoryId, $typeId, $path = null, $order = 're
   private function readIssuesAjaxDB($categoryId, $typeId, $path = null, $order = 'records DESC', $page = 0, $limit = 100) {
     $groupId = $this->grouped ? $this->groupId : '';
     if (is_null($path) || empty($path)) {
-      $this->recordCount = $this->issueDB()->getByCategoryTypeAndGroupCount($categoryId, $typeId, $groupId)->fetchArray(SQLITE3_ASSOC)['count'];
-      $result = $this->issueDB()->getByCategoryTypeAndGroup($categoryId, $typeId, $groupId, $order, $page * $limit, $limit);
+      $this->recordCount = $this->issueDB($this->delta)->getByCategoryTypeAndGroupCount($categoryId, $typeId, $groupId)->fetchArray(SQLITE3_ASSOC)['count'];
+      $result = $this->issueDB($this->delta)->getByCategoryTypeAndGroup($categoryId, $typeId, $groupId, $order, $page * $limit, $limit);
     } else {
-      $this->recordCount = $this->issueDB()->getByCategoryTypePathAndGroupCount($categoryId, $typeId, $path, $groupId)->fetchArray(SQLITE3_ASSOC)['count'];
-      $result = $this->issueDB()->getByCategoryTypePathAndGroup($categoryId, $typeId, $path, $groupId, $order, $page * $limit, $limit);
+      $this->recordCount = $this->issueDB($this->delta)->getByCategoryTypePathAndGroupCount($categoryId, $typeId, $path, $groupId)->fetchArray(SQLITE3_ASSOC)['count'];
+      $result = $this->issueDB($this->delta)->getByCategoryTypePathAndGroup($categoryId, $typeId, $path, $groupId, $order, $page * $limit, $limit);
     }
 
     while ($row = $result->fetchArray(SQLITE3_ASSOC)) {
@@ -174,9 +186,9 @@ private function readIssuesAjaxByTag($categoryId, $typeId, $order = 'records DES
     $groupId = $this->grouped ? $this->groupId : '';
 
     if ($this->grouped) {
-      $result = $this->issueDB()->getRecordNumberAndVariationsForPathGrouped($typeId, $groupId, $order, $page * $limit, $limit);
+      $result = $this->issueDB($this->delta)->getRecordNumberAndVariationsForPathGrouped($typeId, $groupId, $order, $page * $limit, $limit);
     } else {
-      $result = $this->issueDB()->getByCategoryAndTypeGroupedByPath($categoryId, $typeId, $groupId, $order, $page * $limit, $limit);
+      $result = $this->issueDB($this->delta)->getByCategoryAndTypeGroupedByPath($categoryId, $typeId, $groupId, $order, $page * $limit, $limit);
     }
     while ($row = $result->fetchArray(SQLITE3_ASSOC)) {
       $record = (object) $row;
@@ -206,6 +218,8 @@ private function readIssuesAjaxCSV($categoryId, $typeId, $path = null, $order =
     $lineNumber = 0;
     if ($this->versioning && $this->version != '') {
       $elementsFile = $this->getVersionedFilePath($this->version, 'issue-summary.csv');
+    } elseif ($this->delta) {
+      $elementsFile = $this->getDeltaFilePath('issue-summary.csv');
     } else {
       $elementsFile = $this->getFilePath('issue-summary.csv');
     }
@@ -296,6 +310,8 @@ private function readTypes() {
   protected function filePath($filename) {
     if ($this->versioning && $this->version != '') {
       $elementsFile = $this->getVersionedFilePath($this->version, $filename);
+    } elseif ($this->delta) {
+        $elementsFile = $this->getDeltaFilePath($filename);
     } else {
       $elementsFile = $this->getFilePath($filename);
     }
@@ -398,11 +414,11 @@ private function getIds($errorId, $action) {
   private function getIdsFromDBResult($id, $type, $action): SQLite3Result {
     $groupId = $this->grouped ? $this->groupId : '';
     if ($type == 'errorId')
-      $result = $this->issueDB()->getRecordIdsByErrorId($id, $groupId);
+      $result = $this->issueDB($this->delta)->getRecordIdsByErrorId($id, $groupId);
     else if ($type == 'categoryId')
-      $result = $this->issueDB()->getRecordIdsByCategoryId($id, $groupId);
+      $result = $this->issueDB($this->delta)->getRecordIdsByCategoryId($id, $groupId);
     else if ($type == 'typeId')
-      $result = $this->issueDB()->getRecordIdsByTypeId($id, $groupId);
+      $result = $this->issueDB($this->delta)->getRecordIdsByTypeId($id, $groupId);
     else
       $result = false;
 
@@ -424,6 +440,8 @@ private function getIdsFromDB($id, $type, $action) {
   private function getIdsFromCsv($errorId, $action) {
     if ($this->versioning && $this->version != '') {
       $elementsFile = $this->getVersionedFilePath($this->version, 'issue-collector.csv');
+    } elseif ($this->delta) {
+      $elementsFile = $this->getDeltaFilePath('issue-collector.csv');
     } else {
       $elementsFile = $this->getFilePath('issue-collector.csv');
     }
@@ -479,9 +497,11 @@ private function getAjaxParameters(): void {
     $this->limit = getOrDefault('limit', $this->issueLimit);
   }
 
-  protected function getDbDir() : string {
+  protected function getDbDir(bool $delta = false) : string {
     if ($this->versioning && $this->version != '') {
       $dir = sprintf('%s/_historical/%s/%s', $this->configuration->getDir(), $this->configuration->getDirName(), $this->version);
+    } elseif ($this->delta) {
+      $dir = sprintf('%s/%s/delta', $this->configuration->getDir(), $this->configuration->getDirName());
     } else {
       $dir = sprintf('%s/%s', $this->configuration->getDir(), $this->configuration->getDirName());
     }
@@ -671,12 +691,17 @@ private function processListRequest(Smarty $smarty): void {
       $this->filePath('issue-total.csv'),
       $this->count,
       $this->grouped,
-      ($this->grouped ? $this->currentGroup : null)));
+      ($this->grouped ? $this->currentGroup : null))
+    );
     $smarty->assign('total', $this->count);
     $smarty->assign('fieldNames', ['path', 'message', 'instances', 'records']);
     $smarty->assign('listType', 'full-list');
     $smarty->assign('path', null);
     $smarty->assign('order', null);
     $smarty->registerPlugin("function", 'showMarcUrl', array('Issues', 'showMarcUrl'));
   }
+
+  public function setDelta() {
+    $this->delta = true;
+  }
 }
diff --git a/classes/Tab.php b/classes/Tab.php
@@ -18,6 +18,7 @@ abstract class Tab {
     'tt-completeness'          => 'TtCompleteness',
     'shelf-ready-completeness' => 'ShelfReadyCompleteness',
     'shacl'                    => 'Shacl',
+    'delta'                    => 'Issues',
     'network'                  => 'Network',
     'terms'                    => 'Terms',
     'pareto'                   => 'Pareto',
@@ -35,7 +36,11 @@ abstract class Tab {
 
   public static function create($name, $config): Tab {
     $class = Tab::names[$name];
-    return new $class($config, $config->getId()); 
+    $instance = new $class($config, $config->getId());
+    if ($name == 'delta') {
+      $instance->setDelta();
+    }
+    return $instance;
   }
 
   public static function defined($name): bool {

diff --git a/index.php b/index.php
@@ -41,7 +41,7 @@
 
 $smarty->assign('tab', $tab);
 $smarty->assign('isCompleteness', in_array($tab, ['completeness', 'serials', 'tt-completeness', 'shelf-ready-completeness', 'functions']));
-$smarty->assign('isValidation', in_array($tab, ['issues', 'shacl']));
+$smarty->assign('isValidation', in_array($tab, ['issues', 'shacl', 'delta']));
 $smarty->assign('isAuthority', in_array($tab, ['classifications', 'authorities']));
 $smarty->assign('isTool', in_array($tab, ['terms', 'control-fields', 'collocations', 'download', 'settings']));
 $languages = [

diff --git a/templates/common/nav-tabs.tpl b/templates/common/nav-tabs.tpl
@@ -45,16 +45,21 @@
       {/if}
     </li>
     {/if}
-    {if $display['shacl'] || $display['issues']}
+    {if $display['issues'] || $display['shacl'] || $display['delta']}
     <li class="nav-item1">
-      {if $display['shacl']}
+      {if $display['shacl'] || $display['delta']}
         <a class="nav-link1 {if $isValidation}active{/if} dropdown-toggle"
            data-toggle="dropdown" role="tab1" aria-selected="true"
            id="validation-tab" aria-controls="validation"
            href="#">{_('Validation')}</a>
         <div class="dropdown-menu">
           <a class="dropdown-item" href="?tab=issues{$generalParams}{if isset($groupId)}&groupId={$groupId}{/if}">{_('Issues')}</a>
-          <a class="dropdown-item" href="?tab=shacl{$generalParams}{if isset($groupId)}&groupId={$groupId}{/if}">{_('Custom validation')}</a>
+          {if $display['delta']}
+            <a class="dropdown-item" href="?tab=delta{$generalParams}{if isset($groupId)}&groupId={$groupId}{/if}">{_('Validation of latest changes')}</a>
+          {/if}
+          {if $display['shacl']}
+            <a class="dropdown-item" href="?tab=shacl{$generalParams}{if isset($groupId)}&groupId={$groupId}{/if}">{_('Custom validation')}</a>
+          {/if}
         </div>
       {elseif $display['issues']}
         <a class="nav-link1 {if $tab == 'issues'}active{/if}" data-toggle="tab1" role="tab1" aria-selected="false"

diff --git a/templates/issues/issues.tpl b/templates/issues/issues.tpl
@@ -4,7 +4,11 @@
   {include 'common/nav-tabs.tpl'}
   <div class="tab-content" id="myTabContent">
     <div class="tab-pane active" id="issues" role="tabpanel" aria-labelledby="issues-tab">
-      <h2>{_('Violations of record format specification')}</h2>
+      {if $delta}
+        <h2>{_('Validation of latest changes')}</h2>
+      {else}
+        <h2>{_('Violations of record format specification')}</h2>
+      {/if}
 
       <p class="metric-definition">
         {_t('issues_definition')}
@@ -27,6 +31,11 @@
           {/foreach}
         </p>
       {/if}
+
+      {if $delta}
+        <p><span style="color: #999999">number of new/updated reords:</span> {$deltaCount}</p>
+      {/if}
+
       <div id="issues-table-placeholder">
         {include 'issues/issue-summary.tpl'}
       </div>