From 5ef0500f781f6a4fa2574777f7ca0725af170623 Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Mon, 30 Dec 2024 08:26:37 -0700 Subject: [PATCH 01/10] test: Minor updates to linux-setup.sh (#6869) [skip ci] Co-authored-by: Stanislav Zhuk --- .github/workflows/linux-setup.sh | 28 +++++++----------------- .github/workflows/selfhosted-upgrades.sh | 2 +- pkg/ddevapp/ddevapp_test.go | 7 +----- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/.github/workflows/linux-setup.sh b/.github/workflows/linux-setup.sh index 25346ca6abd5..39e0e6e55afd 100755 --- a/.github/workflows/linux-setup.sh +++ b/.github/workflows/linux-setup.sh @@ -13,38 +13,26 @@ if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then fi sudo apt-get update -qq >/dev/null -sudo apt-get install -qq zip jq expect nfs-kernel-server build-essential curl git libnss3-tools libcurl4-gnutls-dev postgresql-client >/dev/null +sudo apt-get install -y -qq build-essential expect libnss3-tools libcurl4-gnutls-dev postgresql-client >/dev/null -curl -sSL --fail -o /tmp/ngrok.zip https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip && sudo unzip -o -d /usr/local/bin /tmp/ngrok.zip - -export PATH=/home/linuxbrew/.linuxbrew/bin:$PATH -echo "export PATH=/home/linuxbrew/.linuxbrew/bin:$PATH" >>~/.bashrc +curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc \ + | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null \ + && echo "deb https://ngrok-agent.s3.amazonaws.com buster main" \ + | sudo tee /etc/apt/sources.list.d/ngrok.list \ + && sudo apt-get update -qq >/dev/null \ + && sudo apt-get install -y -qq ngrok # Without this .curlrc CircleCI linux image doesn't respect mkcert certs echo "capath=/etc/ssl/certs/" >>~/.curlrc source ~/.bashrc -for item in bats-core ddev/ddev/ddev docker-compose golangci-lint kaos/shell/bats-assert kaos/shell/bats-file mkcert; do +for item in bats-core ddev/ddev/ddev docker-compose ghr golangci-lint kaos/shell/bats-assert kaos/shell/bats-file; do brew install $item >/dev/null || brew upgrade $item >/dev/null done mkcert -install -primary_ip=$(ip route get 1 | awk '{gsub("^.*src ",""); print $1; exit}') - -sudo bash -c "cat </etc/exports -${HOME} ${primary_ip}/255.255.255.255(rw,sync,no_subtree_check) -/tmp ${primary_ip}/255.255.255.255(rw,sync,no_subtree_check) -EOF" - -sudo service nfs-kernel-server restart - -# Install ghr -GHR_RELEASE="v0.14.0" -curl -fsL -o /tmp/ghr.tar.gz https://github.com/tcnksm/ghr/releases/download/${GHR_RELEASE}/ghr_${GHR_RELEASE}_linux_amd64.tar.gz -sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/ghr.tar.gz ghr_${GHR_RELEASE}_linux_amd64/ghr - # Show info to simplify debugging docker info docker version diff --git a/.github/workflows/selfhosted-upgrades.sh b/.github/workflows/selfhosted-upgrades.sh index 59d764a6f560..4945dbd4c57f 100755 --- a/.github/workflows/selfhosted-upgrades.sh +++ b/.github/workflows/selfhosted-upgrades.sh @@ -22,7 +22,7 @@ fi # Upgrade various items on various operating systems case $os in darwin) - for item in mkcert mkdocs golang golangci-lint ddev; do + for item in mkdocs golang golangci-lint ddev; do brew upgrade $item || brew install $item || true done ;; diff --git a/pkg/ddevapp/ddevapp_test.go b/pkg/ddevapp/ddevapp_test.go index 873c452b9ebb..3fd727698de9 100644 --- a/pkg/ddevapp/ddevapp_test.go +++ b/pkg/ddevapp/ddevapp_test.go @@ -3818,12 +3818,7 @@ func TestCaptureLogs(t *testing.T) { // This requires that the test machine must have NFS shares working // Tests using both app-specific performance_mode: nfs and etc func TestNFSMount(t *testing.T) { - if nodeps.IsWSL2() || dockerutil.IsColima() || dockerutil.IsLima() || dockerutil.IsOrbstack() { - t.Skip("Skipping on WSL2/Lima/Colima") - } - if nodeps.PerformanceModeDefault == types.PerformanceModeMutagen || nodeps.NoBindMountsDefault { - t.Skip("Skipping because mutagen/nobindmounts enabled") - } + t.Skip("Skipping because NFS is deprecated") assert := asrt.New(t) app := &ddevapp.DdevApp{} From 1d1ce7a471406658bb4febae42221a70956d013e Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Mon, 30 Dec 2024 08:28:12 -0700 Subject: [PATCH 02/10] test: get more disk space on hosted runner, switch to ubuntu 24.04, fixes #6865 (#6866) --- .github/workflows/pr-build.yml | 2 +- .github/workflows/push-tagged-dbimage.yml | 2 +- .github/workflows/tests.yml | 11 ++++++-- pkg/ddevapp/ddevapp_test.go | 7 +++++ pkg/ddevapp/providerAcquia_test.go | 9 ++++--- pkg/ddevapp/providerLagoon_test.go | 16 ++++++------ pkg/ddevapp/providerPantheon_test.go | 1 + pkg/testcommon/testcommon.go | 2 ++ pkg/testcommon/testcommon_test.go | 31 ++++++++++------------- 9 files changed, 48 insertions(+), 33 deletions(-) diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 61f72d31b384..aea1b76c88b4 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -33,7 +33,7 @@ permissions: jobs: build: name: Build DDEV executables - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: AmplitudeAPIKey: ${{ secrets.AMPLITUDE_API_KEY_DEV }} steps: diff --git a/.github/workflows/push-tagged-dbimage.yml b/.github/workflows/push-tagged-dbimage.yml index 76860b0b5ff6..f501bd8561cd 100644 --- a/.github/workflows/push-tagged-dbimage.yml +++ b/.github/workflows/push-tagged-dbimage.yml @@ -27,7 +27,7 @@ permissions: jobs: push-tagged-dbimage: name: "push tagged dbimage" - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: dbtype: [mariadb_5.5, mariadb_10.0, mariadb_10.1, mariadb_10.2, mariadb_10.3, mariadb_10.4, mariadb_10.5, mariadb_10.6, mariadb_10.7, mariadb_10.8, mariadb_10.11, mariadb_11.4, mysql_5.5, mysql_5.6, mysql_5.7, mysql_8.0, mysql_8.4] diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b8a86de4bca5..25fe86a2c653 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 env: CGO_ENABLED: 0 @@ -109,7 +109,14 @@ jobs: uses: Homebrew/actions/setup-homebrew@master - name: Remove unnecessary items on disk - run: sudo rm -rf /usr/local/lib/android && df -h . + run: | + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo rm -rf /usr/local/{aws*,julia*} + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/local/lib/node_modules + sudo rm -rf /usr/local/bin/minikube,bin/terraform,bin/oc + sudo rm -rf /usr/local/share/chromium /usr/local/share/powershell /usr/local/share/vcpkg + df -h . - name: Install Docker and deps (Linux) run: ./.github/workflows/linux-setup.sh diff --git a/pkg/ddevapp/ddevapp_test.go b/pkg/ddevapp/ddevapp_test.go index 3fd727698de9..8db016a6b7a8 100644 --- a/pkg/ddevapp/ddevapp_test.go +++ b/pkg/ddevapp/ddevapp_test.go @@ -56,6 +56,7 @@ var ( // 1: drupal8 { Name: "TestPkgDrupal8", + Disable: true, SourceURL: "https://ftp.drupal.org/files/projects/drupal-8.9.20.tar.gz", ArchiveInternalExtractionPath: "drupal-8.9.20/", FilesTarballURL: "https://github.com/ddev/ddev_test_tarballs/releases/download/v1.1/d8_umami.files.tar.gz", @@ -87,6 +88,7 @@ var ( // 3: drupal6 { Name: "TestPkgDrupal6", + Disable: true, SourceURL: "https://ftp.drupal.org/files/projects/drupal-6.38.tar.gz", ArchiveInternalExtractionPath: "drupal-6.38/", DBTarURL: "https://github.com/ddev/ddev_test_tarballs/releases/download/v1.1/drupal6.38_db.tar.gz", @@ -167,6 +169,7 @@ var ( // 8: drupal9 { Name: "TestPkgDrupal9", + Disable: true, SourceURL: "https://ftp.drupal.org/files/projects/drupal-9.5.10.tar.gz", ArchiveInternalExtractionPath: "drupal-9.5.10/", FilesTarballURL: "https://github.com/ddev/ddev_test_tarballs/releases/download/v1.1/d9_umami_files.tgz", @@ -2290,6 +2293,10 @@ func TestDdevFullSiteSetup(t *testing.T) { app := &ddevapp.DdevApp{} for i, site := range TestSites { + if site.Disable { + t.Logf("Skipping TestSite %s=%d because disabled", site.Name, i) + continue + } switchDir := site.Chdir() defer switchDir() runTime := util.TimeTrackC(fmt.Sprintf("%s DdevFullSiteSetup", site.Name)) diff --git a/pkg/ddevapp/providerAcquia_test.go b/pkg/ddevapp/providerAcquia_test.go index fc73a7aedc48..c4b78c70100d 100644 --- a/pkg/ddevapp/providerAcquia_test.go +++ b/pkg/ddevapp/providerAcquia_test.go @@ -88,8 +88,8 @@ func TestAcquiaPull(t *testing.T) { globalconfig.DdevGlobalConfig.WebEnvironment = webEnvSave err = globalconfig.WriteGlobalConfig(globalconfig.DdevGlobalConfig) assert.NoError(err) - _ = os.Chdir(origDir) + _ = os.RemoveAll(app.AppRoot) }) app.Name = t.Name() @@ -117,10 +117,11 @@ func TestAcquiaPull(t *testing.T) { err = app.Pull(provider, false, false, false) require.NoError(t, err) - assert.FileExists(filepath.Join(app.GetHostUploadDirFullPath(), "chocolate-brownie-umami.jpg")) + require.FileExists(t, filepath.Join(app.GetHostUploadDirFullPath(), "chocolate-brownie-umami.jpg")) out, err := exec.RunCommand("bash", []string{"-c", fmt.Sprintf(`echo 'select COUNT(*) from users_field_data where mail="randy@example.com";' | %s mysql -B --skip-column-names `, DdevBin)}) - assert.NoError(err) - assert.True(strings.HasSuffix(out, "\n1\n"), "out is unexpected '%s'", out) + require.NoError(t, err) + out = strings.Trim(out, " \n") + require.Equal(t, "1", out) } // TestAcquiaPush ensures we can push to acquia for a configured environment. diff --git a/pkg/ddevapp/providerLagoon_test.go b/pkg/ddevapp/providerLagoon_test.go index 1e5c3a1ff8e7..fbe8c64ea866 100644 --- a/pkg/ddevapp/providerLagoon_test.go +++ b/pkg/ddevapp/providerLagoon_test.go @@ -43,7 +43,6 @@ func lagoonSetupSSHKey(t *testing.T) string { // TestLagoonPull ensures we can pull from lagoon func TestLagoonPull(t *testing.T) { - assert := asrt.New(t) var err error sshKey := lagoonSetupSSHKey(t) @@ -58,9 +57,9 @@ func TestLagoonPull(t *testing.T) { require.NoError(t, err) err = os.Chdir(siteDir) - assert.NoError(err) + require.NoError(t, err) app, err := ddevapp.NewApp(siteDir, true) - assert.NoError(err) + require.NoError(t, err) app.Name = t.Name() app.Type = nodeps.AppTypeDrupal11 err = app.Stop(true, false) @@ -72,10 +71,10 @@ func TestLagoonPull(t *testing.T) { t.Cleanup(func() { err = app.Stop(true, false) - assert.NoError(err) + require.NoError(t, err) err = os.Chdir(origDir) - assert.NoError(err) + require.NoError(t, err) _ = os.RemoveAll(siteDir) }) @@ -107,10 +106,11 @@ func TestLagoonPull(t *testing.T) { err = app.Pull(provider, false, false, false) require.NoError(t, err) - assert.FileExists(filepath.Join(app.GetHostUploadDirFullPath(), "victoria-sponge-umami.jpg")) + require.FileExists(t, filepath.Join(app.GetHostUploadDirFullPath(), "victoria-sponge-umami.jpg")) out, err := exec.RunHostCommand("bash", "-c", fmt.Sprintf(`echo 'select COUNT(*) from users_field_data where mail="margaret.hopper@example.com";' | %s mysql -N`, DdevBin)) - assert.NoError(err) - assert.True(strings.HasSuffix(out, "\n1\n")) + require.NoError(t, err) + out = strings.Trim(out, " \n") + require.Equal(t, "1", out) } // TestLagoonPush ensures we can push to lagoon for a configured environment. diff --git a/pkg/ddevapp/providerPantheon_test.go b/pkg/ddevapp/providerPantheon_test.go index c3ad9aed4009..63ad873167da 100644 --- a/pkg/ddevapp/providerPantheon_test.go +++ b/pkg/ddevapp/providerPantheon_test.go @@ -171,6 +171,7 @@ func TestPantheonPush(t *testing.T) { assert.NoError(err) _ = os.Chdir(origDir) + _ = os.RemoveAll(app.AppRoot) }) app.Name = t.Name() diff --git a/pkg/testcommon/testcommon.go b/pkg/testcommon/testcommon.go index 1482d9cd1009..489ed115ec4a 100644 --- a/pkg/testcommon/testcommon.go +++ b/pkg/testcommon/testcommon.go @@ -43,6 +43,8 @@ type URIWithExpect struct { type TestSite struct { // Name is the generic name of the site, and is used as the default dir. Name string + // Provide ability to disable + Disable bool // SourceURL is the URL of the source code tarball to be used for building the site. SourceURL string // ArchiveExtractionPath is the relative path within the tarball which should be extracted, ending with / diff --git a/pkg/testcommon/testcommon_test.go b/pkg/testcommon/testcommon_test.go index c2719331352e..692ca9127381 100644 --- a/pkg/testcommon/testcommon_test.go +++ b/pkg/testcommon/testcommon_test.go @@ -2,6 +2,12 @@ package testcommon import ( "fmt" + "os" + "path/filepath" + "runtime" + "testing" + "time" + "github.com/ddev/ddev/pkg/ddevapp" "github.com/ddev/ddev/pkg/dockerutil" "github.com/ddev/ddev/pkg/exec" @@ -9,11 +15,6 @@ import ( "github.com/ddev/ddev/pkg/nodeps" asrt "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "os" - "path/filepath" - "runtime" - "testing" - "time" ) var DdevBin = "ddev" @@ -192,33 +193,29 @@ func TestGetLocalHTTPResponse(t *testing.T) { // TestGetCachedArchive tests download and extraction of archives for test sites // to testcache directory. func TestGetCachedArchive(t *testing.T) { - assert := asrt.New(t) - sourceURL := "https://raw.githubusercontent.com/ddev/ddev/master/.gitignore" exPath, archPath, err := GetCachedArchive("TestInvalidArchive", "test", "", sourceURL) - assert.Error(err) + require.Error(t, err) if err != nil { - assert.Contains(err.Error(), fmt.Sprintf("archive extraction of %s failed", archPath)) + require.Contains(t, err.Error(), fmt.Sprintf("archive extraction of %s failed", archPath)) } err = os.RemoveAll(exPath) - assert.NoError(err) + require.NoError(t, err) err = os.RemoveAll(archPath) - assert.NoError(err) + require.NoError(t, err) sourceURL = "http://invalid_domain/somefilethatdoesnotexists" exPath, archPath, err = GetCachedArchive("TestInvalidDownloadURL", "test", "", sourceURL) - assert.Error(err) - if err != nil { - assert.Contains(err.Error(), fmt.Sprintf("failed to download url=%s into %s", sourceURL, archPath)) - } + require.Error(t, err) + require.Contains(t, err.Error(), fmt.Sprintf("failed to download url=%s into %s", sourceURL, archPath)) err = os.RemoveAll(exPath) - assert.NoError(err) + require.NoError(t, err) err = os.RemoveAll(archPath) - assert.NoError(err) + require.NoError(t, err) } // TestPretestAndEnv tests that the testsite PretestCmd works along with WebEvironment From 42681444a9d8f39c98560bed1fb10505cf4b319b Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Mon, 30 Dec 2024 09:47:38 -0700 Subject: [PATCH 03/10] build: dependabot complaint about net/html (#6867) [skip ci] --- go.mod | 2 +- go.sum | 4 ++-- vendor/modules.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 7ea5d9bd803d..c42b5300b304 100644 --- a/go.mod +++ b/go.mod @@ -81,7 +81,7 @@ require ( go.opentelemetry.io/otel/sdk v1.22.0 // indirect go.opentelemetry.io/otel/trace v1.32.0 // indirect golang.org/x/crypto v0.31.0 // indirect - golang.org/x/net v0.31.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/protobuf v1.35.2 // indirect diff --git a/go.sum b/go.sum index cbb30e21961b..16bb074bc1d8 100644 --- a/go.sum +++ b/go.sum @@ -203,8 +203,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/vendor/modules.txt b/vendor/modules.txt index e89be42f2287..f02d3958b16c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -308,7 +308,7 @@ golang.org/x/crypto/sha3 # golang.org/x/mod v0.22.0 ## explicit; go 1.22.0 golang.org/x/mod/semver -# golang.org/x/net v0.31.0 +# golang.org/x/net v0.33.0 ## explicit; go 1.18 # golang.org/x/oauth2 v0.24.0 ## explicit; go 1.18 From d5a17c0ea60331095c3e5eba61179709a4160c8a Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Mon, 30 Dec 2024 09:48:01 -0700 Subject: [PATCH 04/10] test: Add bats test coverage for backdrop quickstart, for #6222 (#6868) Co-authored-by: Stanislav Zhuk --- .github/workflows/quickstart.yml | 55 +++++++++++++++++++++++ Makefile | 5 +++ docs/content/users/quickstart.md | 14 ++++-- docs/tests/backdrop.bats | 75 ++++++++++++++++++++++++++++++++ docs/tests/common-setup.bash | 18 ++++++++ pkg/ddevapp/ddevapp_test.go | 12 ++--- 6 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/quickstart.yml create mode 100644 docs/tests/backdrop.bats create mode 100644 docs/tests/common-setup.bash diff --git a/.github/workflows/quickstart.yml b/.github/workflows/quickstart.yml new file mode 100644 index 000000000000..0f6ccfe40686 --- /dev/null +++ b/.github/workflows/quickstart.yml @@ -0,0 +1,55 @@ +name: Quickstart test +defaults: + run: + shell: bash + +on: + push: + branches: + - master + pull_request: + paths: + - "docs/content/users/quickstart.md" + workflow_dispatch: + inputs: + debug_enabled: + description: 'Enable debug mode' + type: boolean + required: false + default: false + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +permissions: + actions: write + +jobs: + build: + name: Docs Quickstart test + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Install Docker and deps (Linux) + run: ./.github/workflows/linux-setup.sh + + - name: Setup tmate session + if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + + - name: Run quickstart test + run: | + make quickstart-test + diff --git a/Makefile b/Makefile index 41ab2feba8c6..18f29ca11471 100644 --- a/Makefile +++ b/Makefile @@ -286,6 +286,11 @@ golangci-lint: echo "Skipping golangci-lint as not installed"; \ fi +quickstart-test: build + @echo "quickstart-test:" + @echo DDEV_BINARY_FULLPATH=$(DDEV_BINARY_FULLPATH) + export PATH="$(DDEV_PATH):$$PATH" DDEV_NO_INSTRUMENTATION=true CGO_ENABLED=$(CGO_ENABLED) DDEV_BINARY_FULLPATH=$(DDEV_BINARY_FULLPATH); bats docs/tests + version: @echo VERSION:$(VERSION) diff --git a/docs/content/users/quickstart.md b/docs/content/users/quickstart.md index 0fb6a3868949..a5f947c145eb 100644 --- a/docs/content/users/quickstart.md +++ b/docs/content/users/quickstart.md @@ -26,18 +26,24 @@ To get started with [Backdrop](https://backdropcms.org), clone the project repos ```bash # Clone an existing repository (or navigate to a local project directory): - git clone https://github.com/example/example-site my-backdrop-site + # Set PROJECT_GIT_URL to your project's git URL. + PROJECT_GIT_URL=https://github.com/ddev/test-backdrop.git + git clone ${PROJECT_GIT_URL} my-backdrop-site cd my-backdrop-site # Set up the DDEV environment: ddev config --project-type=backdrop - # Boot the project and install Composer packages (if required): + # Start the project ddev start - ddev composer install - # Import a database backup and open the site in your browser: + # Import a database backup: ddev import-db --file=/path/to/db.sql.gz + + # Import files backup + ddev import-files --source=/path/to/files.tar.gz + + # open the site in your browser ddev launch ``` diff --git a/docs/tests/backdrop.bats b/docs/tests/backdrop.bats new file mode 100644 index 000000000000..b50894d0f120 --- /dev/null +++ b/docs/tests/backdrop.bats @@ -0,0 +1,75 @@ +#!/usr/bin/env bats + +setup() { + PROJNAME=my-backdrop-site + load 'common-setup' + _common_setup +} + +# executed after each test +teardown() { + _common_teardown +} + +@test "backdrop new-project quickstart with $(ddev --version)" { + # mkdir my-backdrop-site && cd my-backdrop-site + run mkdir -p my-backdrop-site && cd my-backdrop-site + assert_success + # curl -LJO https://github.com/backdrop/backdrop/releases/latest/download/backdrop.zip + run curl -LJO https://github.com/backdrop/backdrop/releases/latest/download/backdrop.zip + assert_success + # unzip ./backdrop.zip && rm -f backdrop.zip && mv -f ./backdrop/{.,}* . ; rm -rf backdrop + run unzip -o ./backdrop.zip && rm -f backdrop.zip && mv -f ./backdrop/{.,}* . ; rm -rf backdrop + assert_success + # ddev config --project-type=backdrop + run ddev config --project-type=backdrop + assert_success + # ddev start + run ddev start + assert_success + # ddev launch + run bash -c "DDEV_DEBUG=true ddev launch" + assert_output "FULLURL https://${PROJNAME}.ddev.site" + assert_success + # validate running project + run curl -sfI https://${PROJNAME}.ddev.site + assert_success + assert_output --partial "location: https://${PROJNAME}.ddev.site/core/install.php" + assert_output --partial "HTTP/2 302" +} + +@test "backdrop existing project with $(ddev --version)" { + # PROJECT_GIT_URL=https://github.com/ddev/test-backdrop.git + PROJECT_GIT_URL=https://github.com/ddev/test-backdrop.git + # git clone ${PROJECT_GIT_URL} my-backdrop-site + run git clone ${PROJECT_GIT_URL} ${PROJNAME} + assert_success + # cd my-backdrop-site + cd ${PROJNAME} || exit 2 + assert_success + # ddev config --project-type=backdrop + run ddev config --project-type=backdrop + assert_success + # ddev start + run ddev start + assert_success + run curl -fLO https://github.com/ddev/test-backdrop/releases/download/1.29.2/db.sql.gz + assert_success + # ddev import-db --file=/path/to/db.sql.gz + run ddev import-db --file=db.sql.gz + assert_success + run curl -fLO https://github.com/ddev/test-backdrop/releases/download/1.29.2/files.tgz + assert_success + # ddev import-files --source=/path/to/files.tar.gz + run ddev import-files --source=files.tgz + # ddev launch + run bash -c "DDEV_DEBUG=true ddev launch" + assert_output "FULLURL https://${PROJNAME}.ddev.site" + assert_success + run curl -sfI https://${PROJNAME}.ddev.site + assert_success + assert_output --partial "HTTP/2 200" + run curl -sf https://${PROJNAME}.ddev.site + assert_success + assert_output --partial "Welcome to My Backdrop Site!" +} diff --git a/docs/tests/common-setup.bash b/docs/tests/common-setup.bash new file mode 100644 index 000000000000..473e9ce502f5 --- /dev/null +++ b/docs/tests/common-setup.bash @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +_common_setup() { + TEST_BREW_PREFIX="$(brew --prefix 2>/dev/null || true)" + export BATS_LIB_PATH="${BATS_LIB_PATH}:${TEST_BREW_PREFIX}/lib:/usr/lib/bats" + bats_load_library bats-support + bats_load_library bats-assert + mkdir -p ~/tmp + tmpdir=$(mktemp -d ~/tmp/${PROJNAME}.XXXXXX) + export DDEV_NO_INSTRUMENTATION=true + mkdir -p ${tmpdir} && cd ${tmpdir} || exit 1 + ddev delete -Oy ${PROJNAME:-notset} +} + +_common_teardown() { + ddev delete -Oy ${PROJNAME} + rm -rf ${tmpdir} +} diff --git a/pkg/ddevapp/ddevapp_test.go b/pkg/ddevapp/ddevapp_test.go index 8db016a6b7a8..a113758663f9 100644 --- a/pkg/ddevapp/ddevapp_test.go +++ b/pkg/ddevapp/ddevapp_test.go @@ -103,16 +103,16 @@ var ( // 4: backdrop { Name: "TestPkgBackdrop", - SourceURL: "https://github.com/backdrop/backdrop/archive/1.22.0.tar.gz", - ArchiveInternalExtractionPath: "backdrop-1.22.0/", - DBTarURL: "https://github.com/ddev/ddev_test_tarballs/releases/download/v1.1/backdrop_db.11.0.tar.gz", - FilesTarballURL: "https://github.com/ddev/ddev_test_tarballs/releases/download/v1.1/backdrop_files.11.0.tar.gz", + SourceURL: "https://github.com/backdrop/backdrop/archive/1.29.2.tar.gz", + ArchiveInternalExtractionPath: "backdrop-1.29.2/", + DBTarURL: "https://github.com/ddev/test-backdrop/releases/download/1.29.2/db.sql.tar.gz", + FilesTarballURL: "https://github.com/ddev/test-backdrop/releases/download/1.29.2/files.tgz", FullSiteTarballURL: "", Docroot: "", Type: nodeps.AppTypeBackdrop, Safe200URIWithExpectation: testcommon.URIWithExpect{URI: "/README.md", Expect: "Backdrop is a full-featured content management system"}, - DynamicURI: testcommon.URIWithExpect{URI: "/posts/first-post-all-about-kittens", Expect: "Lots of kittens are a good thing"}, - FilesImageURI: "/files/styles/large/public/field/image/kittens-large.jpg", + DynamicURI: testcommon.URIWithExpect{URI: "/posts/your-first-post", Expect: "This is your first post! You may edit or delete it."}, + FilesImageURI: "/files/styles/card/public/field/image/card1-layout.png", }, // 5: typo3 { From ea6a520dcb1b8319cc22d61817fc8aea88e1ceb9 Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Mon, 30 Dec 2024 17:51:23 -0700 Subject: [PATCH 05/10] docs: add setup step for macOS test runners (full disk access) [skip ci] (#6871) --- docs/content/developers/buildkite-testmachine-setup.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/content/developers/buildkite-testmachine-setup.md b/docs/content/developers/buildkite-testmachine-setup.md index 6a0be139dd15..de19b769f242 100644 --- a/docs/content/developers/buildkite-testmachine-setup.md +++ b/docs/content/developers/buildkite-testmachine-setup.md @@ -160,6 +160,8 @@ We are using [Buildkite](https://buildkite.com/ddev) for Windows and macOS testi PATH=$PATH:/usr/local/bin:/opt/homebrew/bin ``` +30. In macOS Settings visit "full disk access" and grant access to `buildkite-agent`, `docker`, `iterm`, `orbstack`. This may prevent startup modal dialogs that prevent `buildkite-agent` or `docker` from continuing properly. + ## Additional Colima macOS setup 1. `brew install colima` From c6504bf220197c3f3a810819448b3f2da30c8dc4 Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Thu, 18 Jul 2024 09:39:15 -0600 Subject: [PATCH 06/10] fix: Support node.js back-end --- .../nginx-site-nodejs.conf | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf diff --git a/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf b/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf new file mode 100644 index 000000000000..d6bd7fe69558 --- /dev/null +++ b/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf @@ -0,0 +1,73 @@ +# ddev nodejs config + +#ddev-generated +# If you want to take over this file and customize it, remove the line above +# and ddev will respect it and won't overwrite the file. +# See https://ddev.readthedocs.io/en/stable/users/extend/customization-extendibility/#custom-nginx-configuration + +server { + listen 80 default_server; + listen 443 ssl default_server; + + root {{ .Docroot }}; + + ssl_certificate /etc/ssl/certs/master.crt; + ssl_certificate_key /etc/ssl/certs/master.key; + + include /etc/nginx/monitoring.conf; + + index index.php index.htm index.html; + + # Disable sendfile as per https://docs.vagrantup.com/v2/synced-folders/virtualbox.html + sendfile off; + error_log /dev/stdout info; + access_log /var/log/nginx/access.log; + + location / { + absolute_redirect off; + try_files $uri $uri/ /index.php?$query_string; # For Drupal >= 7 + } + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_set_header X-Nginx-Proxy true; + proxy_http_version 1.1; + proxy_pass http://127.0.0.1:3000; + } + + # Expire rules for static content + + # Prevent clients from accessing hidden files (starting with a dot) + # This is particularly important if you store .htpasswd files in the site hierarchy + # Access to `/.well-known/` is allowed. + # https://www.mnot.net/blog/2010/04/07/well-known + # https://tools.ietf.org/html/rfc5785 + location ~* /\.(?!well-known\/) { + deny all; + } + + # Prevent clients from accessing to backup/config/source files + location ~* (?:\.(?:bak|conf|dist|fla|in[ci]|log|psd|sh|sql|sw[op])|~)$ { + deny all; + } + + + # Media: images, icons, video, audio, HTC + location ~* \.(jpg|jpeg|gif|png|ico|cur|gz|svg|svgz|mp4|ogg|ogv|webm|webp|htc)$ { + try_files $uri @rewrite; + expires max; + log_not_found off; + } + + # js and css always loaded + location ~* \.(js|css)$ { + try_files $uri @rewrite; + expires -1; + log_not_found off; + } + + include /etc/nginx/common.d/*.conf; + include /mnt/ddev_config/nginx/*.conf; +} From f57152c6b4f1660145f597bed6da6939a54517ba Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Thu, 18 Jul 2024 10:25:11 -0600 Subject: [PATCH 07/10] Add webserver type and project type --- .../etc/supervisor/nodejs.conf | 8 ++++++++ .../etc/supervisor/supervisord-nginx-nodejs.conf | 11 +++++++++++ .../webserver_config_assets/nginx-site-nodejs.conf | 3 +++ pkg/nodeps/values.go | 11 +++++++---- 4 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/nodejs.conf create mode 100644 containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/supervisord-nginx-nodejs.conf diff --git a/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/nodejs.conf b/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/nodejs.conf new file mode 100644 index 000000000000..dfef5d9e7fad --- /dev/null +++ b/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/nodejs.conf @@ -0,0 +1,8 @@ +[program:nodejs] +command = bash -c "cd /var/www/html/${DDEV_DOCROOT} && npm run dev" +priority=5 +stdout_logfile=/var/tmp/logpipe +stdout_logfile_maxbytes=0 +redirect_stderr=true +autorestart=true +startretries=3 diff --git a/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/supervisord-nginx-nodejs.conf b/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/supervisord-nginx-nodejs.conf new file mode 100644 index 000000000000..4a8d4f876610 --- /dev/null +++ b/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/supervisord-nginx-nodejs.conf @@ -0,0 +1,11 @@ +[include] +files = /etc/supervisor/nodejs.conf /etc/supervisor/conf.d/*.conf + +[program:nginx] +command=/usr/sbin/nginx +priority=10 +stdout_logfile=/var/tmp/logpipe +stdout_logfile_maxbytes=0 +redirect_stderr=true +autorestart=true +startretries=3 diff --git a/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf b/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf index d6bd7fe69558..7ff89bb8a809 100644 --- a/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf +++ b/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf @@ -33,7 +33,10 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $http_host; proxy_set_header X-Nginx-Proxy true; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; proxy_http_version 1.1; + proxy_cache_bypass $http_upgrade; proxy_pass http://127.0.0.1:3000; } diff --git a/pkg/nodeps/values.go b/pkg/nodeps/values.go index c7eb8abed279..b7753d3d3d12 100644 --- a/pkg/nodeps/values.go +++ b/pkg/nodeps/values.go @@ -38,8 +38,9 @@ const ( // Webserver types const ( - WebserverNginxFPM = "nginx-fpm" - WebserverApacheFPM = "apache-fpm" + WebserverNginxFPM = "nginx-fpm" + WebserverApacheFPM = "apache-fpm" + WebserverNginxNodeJS = "nginx-nodejs" ) // ValidOmitContainers is the list of things that can be omitted @@ -81,8 +82,9 @@ var GoroutineLimit = 10 // ValidWebserverTypes should be updated whenever supported webserver types are added or // removed, and should be used to ensure user-supplied values are valid. var ValidWebserverTypes = map[string]bool{ - WebserverNginxFPM: true, - WebserverApacheFPM: true, + WebserverNginxFPM: true, + WebserverApacheFPM: true, + WebserverNginxNodeJS: true, } const AppTypeDrupalLatestStable = AppTypeDrupal11 @@ -102,6 +104,7 @@ const ( // AppTypeDrupal is an alias for "most recent Drupal version" AppTypeDrupal = "drupal" AppTypeLaravel = "laravel" + AppTypeNodeJS = "nodejs" AppTypeSilverstripe = "silverstripe" AppTypeSymfony = "symfony" AppTypeMagento = "magento" From 25d710ebb15c4ae61facd88d4f74c64b2bea6102 Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Thu, 18 Jul 2024 11:55:25 -0600 Subject: [PATCH 08/10] just starting to work --- .../ddev-webserver-base-files/etc/supervisor/nodejs.conf | 2 +- .../ddev-webserver-base-scripts/healthcheck.sh | 9 ++++++++- pkg/ddevapp/apptypes.go | 5 +++++ pkg/ddevapp/nodejs.go | 8 ++++++++ .../webserver_config_assets/nginx-site-nodejs.conf | 5 ----- pkg/versionconstants/versionconstants.go | 2 +- 6 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 pkg/ddevapp/nodejs.go diff --git a/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/nodejs.conf b/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/nodejs.conf index dfef5d9e7fad..3154e80f4395 100644 --- a/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/nodejs.conf +++ b/containers/ddev-webserver/ddev-webserver-base-files/etc/supervisor/nodejs.conf @@ -1,5 +1,5 @@ [program:nodejs] -command = bash -c "cd /var/www/html/${DDEV_DOCROOT} && npm run dev" +command = bash -c "cd /var/www/html && ./node_modules/http-server/bin/http-server -p 3000" priority=5 stdout_logfile=/var/tmp/logpipe stdout_logfile_maxbytes=0 diff --git a/containers/ddev-webserver/ddev-webserver-base-scripts/healthcheck.sh b/containers/ddev-webserver/ddev-webserver-base-scripts/healthcheck.sh index f38b948ad4b8..16fc85e2704b 100755 --- a/containers/ddev-webserver/ddev-webserver-base-scripts/healthcheck.sh +++ b/containers/ddev-webserver/ddev-webserver-base-scripts/healthcheck.sh @@ -31,6 +31,7 @@ done phpstatus="false" htmlaccess="false" mailpit="false" +nodestatus="false" if ls /var/www/html >/dev/null; then htmlaccess="true" @@ -58,7 +59,13 @@ if [ "${DDEV_WEBSERVER_TYPE#*-}" = "fpm" ]; then fi fi -if [ "${phpstatus}" = "true" ] && [ "${mailpit}" = "true" ]; then +if [ "${DDEV_WEBSERVER_TYPE#*-}" = "nodejs" ]; then + gunicornstatus="true" + phpstatus="true" + nodestatus="true" +fi + +if [ "${phpstatus}" = "true" ] && [ "${htmlaccess}" = "true" ] && [ "${mailpit}" = "true" ]; then touch /tmp/healthy exit 0 fi diff --git a/pkg/ddevapp/apptypes.go b/pkg/ddevapp/apptypes.go index 22b28ab69283..406b570157f4 100644 --- a/pkg/ddevapp/apptypes.go +++ b/pkg/ddevapp/apptypes.go @@ -207,6 +207,11 @@ func init() { importFilesAction: magentoImportFilesAction, }, + // TODO: Fill it in. + nodeps.AppTypeNodeJS: { + configOverrideAction: nodejsConfigOverrideAction, + }, + nodeps.AppTypePHP: { postStartAction: nil, }, diff --git a/pkg/ddevapp/nodejs.go b/pkg/ddevapp/nodejs.go new file mode 100644 index 000000000000..f2944f1de6f2 --- /dev/null +++ b/pkg/ddevapp/nodejs.go @@ -0,0 +1,8 @@ +package ddevapp + +import "github.com/ddev/ddev/pkg/nodeps" + +func nodejsConfigOverrideAction(app *DdevApp) error { + app.WebserverType = nodeps.WebserverNginxNodeJS + return nil +} diff --git a/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf b/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf index 7ff89bb8a809..c211af2f2943 100644 --- a/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf +++ b/pkg/ddevapp/webserver_config_assets/nginx-site-nodejs.conf @@ -23,11 +23,6 @@ server { error_log /dev/stdout info; access_log /var/log/nginx/access.log; - location / { - absolute_redirect off; - try_files $uri $uri/ /index.php?$query_string; # For Drupal >= 7 - } - location / { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/pkg/versionconstants/versionconstants.go b/pkg/versionconstants/versionconstants.go index 48d5c5f0a67b..c2472b17cf46 100644 --- a/pkg/versionconstants/versionconstants.go +++ b/pkg/versionconstants/versionconstants.go @@ -11,7 +11,7 @@ var AmplitudeAPIKey = "" var WebImg = "ddev/ddev-webserver" // WebTag defines the default web image tag -var WebTag = "20241223_stasadev_build_warn" // Note that this can be overridden by make +var WebTag = "20240718_rfay_node_backend" // Note that this can be overridden by make // DBImg defines the default db image used for applications. var DBImg = "ddev/ddev-dbserver" From edc4848b43c85b54e23934e11c77f7554391ff4d Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Thu, 18 Jul 2024 12:23:01 -0600 Subject: [PATCH 09/10] update templates.go --- pkg/ddevapp/templates.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ddevapp/templates.go b/pkg/ddevapp/templates.go index 6ebe2d3fce37..1452547d8d2f 100644 --- a/pkg/ddevapp/templates.go +++ b/pkg/ddevapp/templates.go @@ -41,7 +41,7 @@ const ConfigInstructions = ` # "ddev xhprof" to enable Xhprof and "ddev xhprof off" to disable it work better, # as leaving Xhprof enabled all the time is a big performance hit. -# webserver_type: nginx-fpm or apache-fpm +# webserver_type: nginx-fpm, apache-fpm, nginx-nodejs # timezone: Europe/Berlin # If timezone is unset, DDEV will attempt to derive it from the host system timezone From 65662a5d0b11dd4076c0d0a8a455b4b15231a080 Mon Sep 17 00:00:00 2001 From: Randy Fay Date: Tue, 3 Sep 2024 20:33:56 -0400 Subject: [PATCH 10/10] update the ddev-webserver tag [skip ci] --- pkg/nodeps/values.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/nodeps/values.go b/pkg/nodeps/values.go index b7753d3d3d12..cf6fa2193a34 100644 --- a/pkg/nodeps/values.go +++ b/pkg/nodeps/values.go @@ -38,9 +38,9 @@ const ( // Webserver types const ( - WebserverNginxFPM = "nginx-fpm" - WebserverApacheFPM = "apache-fpm" - WebserverNginxNodeJS = "nginx-nodejs" + WebserverNginxFPM = "nginx-fpm" + WebserverApacheFPM = "apache-fpm" + WebserverNginxNodeJS = "nginx-nodejs" ) // ValidOmitContainers is the list of things that can be omitted @@ -82,9 +82,9 @@ var GoroutineLimit = 10 // ValidWebserverTypes should be updated whenever supported webserver types are added or // removed, and should be used to ensure user-supplied values are valid. var ValidWebserverTypes = map[string]bool{ - WebserverNginxFPM: true, - WebserverApacheFPM: true, - WebserverNginxNodeJS: true, + WebserverNginxFPM: true, + WebserverApacheFPM: true, + WebserverNginxNodeJS: true, } const AppTypeDrupalLatestStable = AppTypeDrupal11