diff --git a/.github/workflows/deployment-arm64.yml b/.github/workflows/deployment-arm64.yml index c2d54a8e7a1..87338bcee4c 100644 --- a/.github/workflows/deployment-arm64.yml +++ b/.github/workflows/deployment-arm64.yml @@ -172,11 +172,6 @@ jobs: if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }} shell: bash run: | - mkdir ${{runner.temp}}/sshkey - if [[ -z "${{ secrets.buildJabRefPrivateKey }}" ]]; then - echo "buildJabRefPrivateKey is empty, exiting" - exit 1 - fi rsync -Pavz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync" -e 'ssh -p 9922 -i ~/.ssh/id_rsa' build/distribution/ jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/ - name: Upload to GitHub workflow artifacts store if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }} diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index a24306dda91..636db9ae5d7 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -53,7 +53,11 @@ jobs: displayName: macOS eaJDK: https://files.jabref.org/jdks/jdk-macos-x64.tar.gz runs-on: ${{ matrix.os }} - name: Create installer and portable version for ${{ matrix.displayName }} + outputs: + major: ${{ steps.gitversion.outputs.Major }} + minor: ${{ steps.gitversion.outputs.Minor }} + branchname: ${{ steps.gitversion.outputs.branchName }} + name: ${{ matrix.displayName }} installer and portable version steps: - name: Check secrets presence id: checksecrets @@ -105,7 +109,7 @@ jobs: cat gradle.properties sed -i "s/JavaLanguageVersion.of(20)/JavaLanguageVersion.of(21)/" build.gradle - - name: Setup JDK jabref-fix (ubuntu) + - name: Setup JDK jabref-fix (linux) if: (matrix.os == 'ubuntu-latest') shell: bash run: | @@ -134,14 +138,14 @@ jobs: cat gradle.properties sed -i'.bak' -e "s/JavaLanguageVersion.of(20)/JavaLanguageVersion.of(21)/" build.gradle - - name: Setup OSX key chain (macOS) + - name: Setup macOS key chain if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') uses: apple-actions/import-codesign-certs@v2 with: p12-file-base64: ${{ secrets.OSX_SIGNING_CERT }} p12-password: ${{ secrets.OSX_CERT_PWD }} keychain-password: jabref - - name: Setup OSX key chain on OSX for app id cert (macOS) + - name: Setup macOS key chain for app id cert if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') uses: apple-actions/import-codesign-certs@v2 with: @@ -149,16 +153,9 @@ jobs: p12-password: ${{ secrets.OSX_CERT_PWD }} create-keychain: false keychain-password: jabref - - name: Build runtime image (non-macOS) - if: (matrix.os != 'macos-latest') - run: ./gradlew -i -PprojVersion="${{ steps.gitversion.outputs.AssemblySemVer }}" -PprojVersionInfo="${{ steps.gitversion.outputs.InformationalVersion }}" jlinkZip - - name: Prepare merged jars and modules dir (macos) + - name: Prepare merged jars and modules dir (macOS) if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') run: ./gradlew -i -PprojVersion="${{ steps.gitversion.outputs.AssemblySemVer }}" -PprojVersionInfo="${{ steps.gitversion.outputs.InformationalVersion }}" prepareModulesDir - - name: Build installer (non-macos) - if: (matrix.os != 'macos-latest') - shell: bash - run: ./gradlew -i -PprojVersion="${{ steps.gitversion.outputs.AssemblySemVer }}" -PprojVersionInfo="${{ steps.gitversion.outputs.InformationalVersion }}" jpackage - name: Build dmg (macOS) if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') shell: bash @@ -209,7 +206,11 @@ jobs: --resource-dir buildres/mac \ --file-associations buildres/mac/bibtexAssociations.properties \ --jlink-options --bind-services - - name: Package application image (non-macOS) + - name: Build runtime image and installer (linux, Windows) + if: (matrix.os != 'macos-latest') + shell: bash + run: ./gradlew -i -PprojVersion="${{ steps.gitversion.outputs.AssemblySemVer }}" -PprojVersionInfo="${{ steps.gitversion.outputs.InformationalVersion }}" jpackage jlinkZip + - name: Package application image (linux, Windows) if: (matrix.os != 'macos-latest') shell: bash run: ${{ matrix.archivePortable }} @@ -230,35 +231,47 @@ jobs: ar -m -c -a sdsd jabref_${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}_amd64_repackaged.deb debian-binary control.tar.xz data.tar.xz rm debian-binary control.tar.* data.tar.* mv -f jabref_${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}_amd64_repackaged.deb jabref_${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}_amd64.deb - - name: Upload to builds.jabref.org (ubuntu) - if: (matrix.os == 'ubuntu-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue')) - uses: Pendect/action-rsyncer@v2.0.0 - env: - DEPLOY_KEY: ${{ secrets.buildJabRefPrivateKey }} - with: - flags: -vaz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync" - options: '' - ssh_options: '-p 9922' - src: 'build/distribution/' - dest: jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/ - - name: Upload to GitHub workflow artifacts store (Windows) - if: (matrix.os == 'windows-latest') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue')) - uses: actions/upload-artifact@v3 - with: - name: JabRef-${{ matrix.displayName }} - path: build/distribution - - name: Upload to GitHub workflow artifacts store (macOS) + - name: Setup rsync (macOS) if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue')) + run: brew install rsync + - name: Setup rsync (Windows) + if: (matrix.os == 'windows-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue')) + # We want to have rsync available at this place to avoid uploading and downloading from GitHub artifact store (taking > 5 minutes in total) + # We cannot use "action-rsyncer", because that requires Docker which is unavailable on Windows + # We cannot use "setup-rsync", because that does not work on Windows + # We do not use egor-tensin/setup-cygwin@v4, because it replaces the default shell + run: choco install --no-progress rsync + - name: Setup ssh key + if: (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue')) + run: | + echo "${{ secrets.buildJabRefPrivateKey }}" > sshkey + chmod 600 sshkey + - name: Upload to builds.jabref.org (Windows) + if: (matrix.os == 'windows-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue')) + shell: cmd + # for rsync installed by chocolatey, we need the ssh.exe delivered with that installation + run: | + rsync -rt --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r --itemize-changes --stats --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync" -e 'C:\ProgramData\chocolatey\lib\rsync\tools\bin\ssh.exe -p 9922 -i sshkey -o StrictHostKeyChecking=no' build/distribution/ jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/ + - name: Upload to builds.jabref.org (linux, macOS) + # macOS: Negated condition of "Upload to GitHub workflow artifacts store (macOS)" + # Reason: We either upload the non-notarized files - or notarize the files later (and upload these later) + # needs to be on one line; multi line does not work + if: ${{ (!startsWith(github.ref, 'refs/heads/gh-readonly-queue')) && (steps.checksecrets.outputs.secretspresent == 'YES') && ((matrix.os == 'ubuntu-latest') || ((matrix.os == 'macos-latest') && !((startsWith(github.ref, 'refs/tags/') || inputs.notarization == true)))) }} + shell: bash + run: | + rsync -rt --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r --itemize-changes --stats --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync" -e 'ssh -p 9922 -i sshkey -o StrictHostKeyChecking=no' build/distribution/ jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/ + - name: Upload to GitHub workflow artifacts store (macOS) + if: (matrix.os == 'macos-latest') && (steps.checksecrets.outputs.secretspresent == 'YES') && (startsWith(github.ref, 'refs/tags/') || inputs.notarization == true) uses: actions/upload-artifact@v3 with: # tbn = to-be-notarized name: JabRef-macOS-tbn path: build/distribution - notarize: # outsourced in a separate job to be able to rerun if this fails for timeouts - name: Notarize and package macOS binaries - runs-on: macos-latest + announce: + name: Comment on pull request + runs-on: ubuntu-latest needs: [build] - if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }} + if: ${{ github.event_name == 'pull_request' }} steps: - name: Check secrets presence id: checksecrets @@ -271,59 +284,19 @@ jobs: fi env: BUILDJABREFPRIVATEKEY: ${{ secrets.buildJabRefPrivateKey }} - - name: Fetch all history for all tags and branches - if: steps.checksecrets.outputs.secretspresent == 'YES' - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - show-progress: 'false' - - name: Install GitVersion - if: steps.checksecrets.outputs.secretspresent == 'YES' - uses: gittools/actions/gitversion/setup@v0.10.2 - with: - versionSpec: "5.x" - - name: Run GitVersion - if: steps.checksecrets.outputs.secretspresent == 'YES' - id: gitversion - uses: gittools/actions/gitversion/execute@v0.10.2 - - name: Get macOS binaries + - name: Comment PR if: steps.checksecrets.outputs.secretspresent == 'YES' - uses: actions/download-artifact@master - with: - name: JabRef-macOS-tbn - path: build/distribution/ - - name: Notarize dmg - if: (steps.checksecrets.outputs.secretspresent == 'YES') && (startsWith(github.ref, 'refs/tags/') || inputs.notarization == true) - shell: bash - run: | - xcrun notarytool store-credentials "notarytool-profile" --apple-id "vorstand@jabref.org" --team-id "6792V39SK3" --password "${{ secrets.OSX_NOTARIZATION_APP_PWD }}" - xcrun notarytool submit build/distribution/JabRef-${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}.dmg --keychain-profile "notarytool-profile" --wait - xcrun stapler staple build/distribution/JabRef-${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}.dmg - - name: Notarize pkg - if: (steps.checksecrets.outputs.secretspresent == 'YES') && (startsWith(github.ref, 'refs/tags/') || inputs.notarization == true) - shell: bash - run: | - xcrun notarytool store-credentials "notarytool-profile" --apple-id "vorstand@jabref.org" --team-id "6792V39SK3" --password "${{ secrets.OSX_NOTARIZATION_APP_PWD }}" - xcrun notarytool submit build/distribution/JabRef-${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}.pkg --keychain-profile "notarytool-profile" --wait - xcrun stapler staple build/distribution/JabRef-${{ steps.gitversion.outputs.Major }}.${{ steps.gitversion.outputs.Minor }}.pkg - - name: Package application image - if: (steps.checksecrets.outputs.secretspresent == 'YES') && (matrix.os != 'macos-latest') - shell: bash - run: ${{ matrix.archivePortable }} - - name: Upload to GitHub workflow artifacts store - if: (steps.checksecrets.outputs.secretspresent == 'YES') && (!startsWith(github.ref, 'refs/heads/gh-readonly-queue')) - uses: actions/upload-artifact@v3 + uses: thollander/actions-comment-pull-request@v2 with: - name: JabRef-macOS - path: build/distribution - upload: - strategy: - fail-fast: false - name: Upload binaries on builds.jabref.org - runs-on: ubuntu-latest - needs: [build, notarize] - if: ${{ !startsWith(github.ref, 'refs/heads/gh-readonly-queue') }} + message: | + The build of this PR is available at . + comment_tag: download-link + mode: recreate + notarize: # outsourced in a separate job to be able to rerun if this fails for timeouts + name: macOS notarization + runs-on: macos-latest + needs: [build] + if: ${{ startsWith(github.ref, 'refs/tags/') || inputs.notarization == true }} steps: - name: Check secrets presence id: checksecrets @@ -336,58 +309,30 @@ jobs: fi env: BUILDJABREFPRIVATEKEY: ${{ secrets.buildJabRefPrivateKey }} - - name: Checkout source - if: steps.checksecrets.outputs.secretspresent == 'YES' - uses: actions/checkout@v4 - with: - fetch-depth: 0 - submodules: 'true' - show-progress: 'false' - - name: Install GitVersion - if: steps.checksecrets.outputs.secretspresent == 'YES' - uses: gittools/actions/gitversion/setup@v0.10.2 - with: - versionSpec: '5.x' - - name: Run GitVersion - if: steps.checksecrets.outputs.secretspresent == 'YES' - id: gitversion - uses: gittools/actions/gitversion/execute@v0.10.2 - - name: Get Windows binaries - if: steps.checksecrets.outputs.secretspresent == 'YES' - uses: actions/download-artifact@master - with: - name: JabRef-windows - path: build/distribution - - name: Get macOS binaries unsigned - if: (steps.checksecrets.outputs.secretspresent == 'YES') && (inputs.notarization == false && !startsWith(github.ref, 'refs/tags/')) + - name: Download from GitHub workflow artifacts store (macOS) + if: (steps.checksecrets.outputs.secretspresent == 'YES') uses: actions/download-artifact@master with: name: JabRef-macOS-tbn path: build/distribution/ - - name: Get macOS binaries notarized - if: (steps.checksecrets.outputs.secretspresent == 'YES') && (inputs.notarization == true || startsWith(github.ref, 'refs/tags/')) - uses: actions/download-artifact@master - with: - name: JabRef-macOS - path: build/distribution/ - # Upload to build server using rsync - # The action runs on linux only (because it is a Dockerized action), therefore it is embedded in a separate workflow + - name: Notarize dmg + if: (steps.checksecrets.outputs.secretspresent == 'YES') + shell: bash + run: | + xcrun notarytool store-credentials "notarytool-profile" --apple-id "vorstand@jabref.org" --team-id "6792V39SK3" --password "${{ secrets.OSX_NOTARIZATION_APP_PWD }}" + xcrun notarytool submit build/distribution/JabRef-${{ needs.build.outputs.major }}.${{ needs.build.outputs.minor }}.dmg --keychain-profile "notarytool-profile" --wait + xcrun stapler staple build/distribution/JabRef-${{ needs.build.outputs.major }}.${{ needs.build.outputs.minor }}.dmg + - name: Notarize pkg + if: (steps.checksecrets.outputs.secretspresent == 'YES') + shell: bash + run: | + xcrun notarytool store-credentials "notarytool-profile" --apple-id "vorstand@jabref.org" --team-id "6792V39SK3" --password "${{ secrets.OSX_NOTARIZATION_APP_PWD }}" + xcrun notarytool submit build/distribution/JabRef-${{ needs.build.outputs.major }}.${{ needs.build.outputs.minor }}.pkg --keychain-profile "notarytool-profile" --wait + xcrun stapler staple build/distribution/JabRef-${{ needs.build.outputs.major }}.${{ needs.build.outputs.minor }}.pkg - name: Upload to builds.jabref.org - if: steps.checksecrets.outputs.secretspresent == 'YES' - uses: Pendect/action-rsyncer@v2.0.0 - env: - DEPLOY_KEY: ${{ secrets.buildJabRefPrivateKey }} - with: - flags: -vaz --itemize-changes --stats --partial-dir=/tmp/partial --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }} && rsync" - options: '' - ssh_options: '-p 9922' - src: 'build/distribution/' - dest: jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ steps.gitversion.outputs.branchName }}/ - - name: Comment PR - if: github.event_name == 'pull_request' && steps.checksecrets.outputs.secretspresent == 'YES' - uses: thollander/actions-comment-pull-request@v2 - with: - message: | - The build of this PR is available at . - comment_tag: download-link - mode: recreate + if: (steps.checksecrets.outputs.secretspresent == 'YES') + shell: bash + run: | + echo "${{ secrets.buildJabRefPrivateKey }}" > sshkey + chmod 600 sshkey + rsync -rt --chmod=Du=rwx,Dg=rx,Do=rx,Fu=rw,Fg=r,Fo=r --itemize-changes --stats --rsync-path="mkdir -p /var/www/builds.jabref.org/www/${{ needs.build.outputs.branchname }} && rsync" -e 'ssh -p 9922 -i sshkey -o StrictHostKeyChecking=no' build/distribution/ jrrsync@build-upload.jabref.org:/var/www/builds.jabref.org/www/${{ needs.build.outputs.branchname }}/ diff --git a/.github/workflows/refresh-journal-lists.yml b/.github/workflows/refresh-journal-lists.yml index 340fb4553f3..5ab88581d88 100644 --- a/.github/workflows/refresh-journal-lists.yml +++ b/.github/workflows/refresh-journal-lists.yml @@ -41,7 +41,7 @@ jobs: cache: 'gradle' - name: Check whether journal-list.mv can be generated (the "real" generation is done inside JabRef's build process) run: | - ./gradlew generateJournalAbbreviationList + ./gradlew generateJournalListMV - uses: peter-evans/create-pull-request@v5 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/build.gradle b/build.gradle index ba9434b63f4..17fe64deda5 100644 --- a/build.gradle +++ b/build.gradle @@ -323,14 +323,17 @@ tasks.register("generateSearchGrammarSource", JavaExec) { args = ["-o","src-gen/main/java/org/jabref/search" , "-visitor", "-no-listener", "-package", "org.jabref.search", "$projectDir/src/main/antlr4/org/jabref/search/Search.g4"] } -tasks.register("generateJournalAbbreviationList", JavaExec) { +tasks.register("generateJournalListMV", JavaExec) { group = "JabRef" description = "Converts the comma-separated journal abbreviation file to a H2 MVStore" classpath = sourceSets.main.runtimeClasspath mainClass = "org.jabref.cli.JournalListMvGenerator" + onlyIf { + !file("build/resources/main/journals/journal-list.mv").exists() + } } -jar.dependsOn "generateJournalAbbreviationList" -test.dependsOn "generateJournalAbbreviationList" +jar.dependsOn "generateJournalListMV" +compileTestJava.dependsOn "generateJournalListMV" tasks.register('generateCitaviSource', XjcTask) { group = 'JabRef' @@ -550,6 +553,7 @@ tasks.register('deleteInstallerTemp', Delete) { } jpackage.dependsOn deleteInstallerTemp +jlinkZip.dependsOn jpackage jlink { addOptions('--strip-debug', '--compress', '2', '--no-header-files', '--no-man-pages') launcher { diff --git a/src/main/java/org/jabref/cli/JournalListMvGenerator.java b/src/main/java/org/jabref/cli/JournalListMvGenerator.java index 35f25a6edbe..9a4fd3d67c6 100644 --- a/src/main/java/org/jabref/cli/JournalListMvGenerator.java +++ b/src/main/java/org/jabref/cli/JournalListMvGenerator.java @@ -27,12 +27,6 @@ public static void main(String[] args) throws IOException { System.exit(0); } Path journalListMvFile = Path.of("build", "resources", "main", "journals", "journal-list.mv"); - if (Files.exists(journalListMvFile)) { - System.out.println("Target " + journalListMvFile.toAbsolutePath() + " already exists."); - System.out.println("Skipping generation."); - System.out.println("If you really want to re-generate, please delete the file."); - System.exit(0); - } Set ignoredNames = Set.of( // remove all lists without dot in them: