Fix gh actions integration testing

.github/workflows/php.yml

@@ -2,63 +2,43 @@ name: Build and Test
 
 on:
   push:
-    branches: [ "master" ]
+    branches: [ "main" ]
   pull_request:
-    branches: [ "master" ]
+    branches: [ "main" ]
 
 permissions:
   contents: read
 
 jobs:
   build-and-test:
     runs-on: ubuntu-latest
-    container:
-      image: dgarciabriseno/helioviewer.org
-      options: --user root
-
     steps:
     - uses: actions/checkout@v3
 
-    - name: Cache Composer packages
-      id: composer-cache
-      uses: actions/cache@v3
-      with:
-        path: vendor
-        key: ${{ runner.os }}-php-${{ hashFiles('**/composer.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-php
-
-    - name: Set up test environment
-      run: |
-        mkdir log
-        mkdir cache
-        ln -s $PWD /home/helioviewer/api.helioviewer.org
-        ln -s /tmp/jp2 docroot/jp2
-        composer run-script make-test-config
-
-    - name: Install dependencies
-      run: composer install --prefer-dist --no-progress
-
-    - name: Install python packages for root user
-      run: |
-        su helioviewer -c "python3 -m pip freeze > /tmp/helioviewer-packages"
-        python3 -m pip install -r /tmp/helioviewer-packages
-
-    - name: Startup background services
-      run: |
-        httpd
-        mysqld --user=mysql -D
-        redis-server --daemonize yes
-        tcsh scripts/movie_queue.tcsh
-        ./vendor/bin/start_hgs2hpc
+    - name: Get development docker compose file
+      run: cd .. && wget https://raw.githubusercontent.com/Helioviewer-Project/helioviewer.org-docker/main/compose.yaml
 
-    - name: Re-install Helioviewer Database for current branch
+    # The containers need to be able update config files
+    - name: Make writeable config files
       run: |
-        mysql -e "drop database helioviewer; drop user 'helioviewer'@'localhost';"
-        cd management/data && ./setup_db.exp
-
-    - name: Run PHP test suite
-      run: composer run-script test
-
-    - name: Run Python test suite
-      run: composer run-script test-python
+        touch install/settings/settings.cfg
+        chmod o+rw install/settings/settings.cfg
+        touch settings/Config.ini
+        chmod o+rw settings/Config.ini
+        touch settings/Config.php
+        chmod o+rw settings/Config.php
+
+    # Start up api service (test environment)
+    # Start up cli service (installs test data)
+    - name: Start services
+      # --no-build to force it to use whatever is published in the registry
+      # -d to detach, so the command returns
+      # --wait to wait for containers to be healthy, meaning the test data is downloaded and ready
+      run: cd .. && docker compose up --no-build -d --wait api cli
+
+    # Run the tests inside the api container
+    - name: Run phpunit tests
+      run: docker exec -t helioviewer-api-1 composer run-script test
+
+    - name: Run python tests
+      run: docker exec -t helioviewer-api-1 composer run-script test-python

composer.json

@@ -1,6 +1,6 @@
 {
     "scripts": {
-        "test-python": "cd install && python3 -m pytest",
+        "test-python": "cd install && venv/bin/python -m pytest",
         "test": "vendor/bin/phpunit --bootstrap tests/autoload.php --fail-on-warning --testdox tests/unit_tests",
         "local-test": "vendor/bin/phpunit --bootstrap tests/autoload.php --stop-on-failure --testdox --fail-on-warning tests/unit_tests",
         "run-test": "vendor/bin/phpunit --bootstrap tests/autoload.php --stop-on-failure --testdox --fail-on-warning tests/unit_tests --filter ",

install/.gitignore

@@ -0,0 +1 @@
+venv

install/helioviewer/hvpull/net/daemon.py

@@ -621,7 +621,6 @@ def create_image_directory(self, path):
         """
         if not os.path.exists(path):
             permissions = stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
-            group_id = self.get_helioviewer_group()
             user_id = os.getuid()
             directories = path.split(os.sep)
             # Traverse the directories to be created so that
@@ -637,12 +636,19 @@ def create_image_directory(self, path):
                     # create the directories and set appropriate permissions.
                     if not os.path.exists(fullpath):
                         os.mkdir(fullpath)
-                        os.chown(fullpath, user_id, group_id)
-                        os.chmod(fullpath, mode=permissions)
-                except:
+                        try:
+                            group_id = self.get_helioviewer_group()
+                            os.chown(fullpath, user_id, group_id)
+                            os.chmod(fullpath, mode=permissions)
+                        except Exception as e:
+                            # Not necessarily an error, things ought to still function, but
+                            # admins may have permission to edit these files.
+                            logging.warn(f"Unable to set group permissions on {fullpath}.")
+                except Exception as e:
                     logging.error("Unable to create the directory '" +
                                   fullpath + "'. Please ensure that you "
                                   "have the proper permissions and try again.")
+                    logging.error(f"Error: {str(e)}")
                     # Do not continue if we don't have a directory to place
                     # the files into
                     sys.exit(1)

install/settings/settings.example.cfg

@@ -11,10 +11,10 @@ dbpass = helioviewer
 ; Temporary solution to keep support for both Helioviewer v2 and Helioviewer v3 databases
 ; Leave fields empty if you don't need to support both versions
 [database_v2]
-dbhost_v2 = 
-dbname_v2 = 
-dbuser_v2 = 
-dbpass_v2 = 
+dbhost_v2 =
+dbname_v2 =
+dbuser_v2 =
+dbpass_v2 =
 
 
 [directories]

install/test_requirements.txt

@@ -0,0 +1,38 @@
+aioftp==0.21.4
+aiohttp==3.9.1
+aiosignal==1.3.1
+astropy==6.0.0
+astropy-iers-data==0.2023.12.11.0.31.11
+async-timeout==4.0.3
+attrs==23.1.0
+contourpy==1.2.0
+cycler==0.12.1
+exceptiongroup==1.2.0
+fonttools==4.46.0
+frozenlist==1.4.1
+Glymur==0.12.9.post1
+idna==3.6
+importlib-resources==6.1.1
+iniconfig==2.0.0
+kiwisolver==1.4.5
+lxml==4.9.3
+matplotlib==3.8.2
+multidict==6.0.4
+mysqlclient==2.2.1
+numpy==1.26.2
+packaging==23.2
+parfive==2.0.2
+Pillow==10.1.0
+pluggy==1.3.0
+pyerfa==2.0.1.1
+pyparsing==3.1.1
+pytest==7.4.3
+python-dateutil==2.8.2
+PyYAML==6.0.1
+scipy==1.11.4
+six==1.16.0
+sunpy==5.1.0
+tomli==2.0.1
+tqdm==4.66.1
+yarl==1.9.4
+zipp==3.17.0

src/Movie/HelioviewerMovie.php

@@ -389,6 +389,9 @@ private function _buildDir() {
      * @return string Movie filename
      */
     private function _buildFilename($highQuality=false) {
+        if (is_null($this->startDate)) {
+            $this->_prepDates();
+        }
         $start = str_replace(array(':', '-', ' '), '_', $this->startDate);
         $end   = str_replace(array(':', '-', ' '), '_', $this->endDate);
 

tests/unit_tests/jhelioviewer/HelioviewerJPXImageTest.php

@@ -63,10 +63,10 @@ public function testCacheConditions_NewFrames() {
         sleep(1);
         // Create JPX instance that aligns with the test file.
         $jpx = new Image_JPEG2000_HelioviewerJPXImage(
-                    13, // AIA 304 source
+                    4, // LASCO C2 source
                     // Known time range for the test file.
-                    "2021-06-01 00:01:00",
-                    "2021-06-01 00:03:00",
+                    "2023-12-01 00:00:00",
+                    "2023-12-01 01:00:00",
                     60,
                     false,
                     self::TEST_JPX_NAME);

tests/unit_tests/movies/reQueueMovieTest.php

@@ -32,7 +32,7 @@ final class reQueueMovieTest extends TestCase
      */
     private function _queueTestMovie() {
         // Queue the movie
-        $url = HV_WEB_ROOT_URL . "/v2/queueMovie/?startTime=2021-06-01T00:02:20Z&endTime=2021-06-02T00:02:00Z&layers=[8,1,100]&imageScale=0.6&events=&eventsLabels=false";
+        $url = HV_WEB_ROOT_URL . "?action=queueMovie&startTime=2023-12-01T00:00:00Z&endTime=2023-12-01T01:00:00Z&layers=[4,1,100]&imageScale=0.6&events=&eventsLabels=false";
         $result = file_get_contents($url);
         $data = json_decode($result);
         // Confirm the API request was accepted