Merge pull request #248 from Islandora-Devops/fixing_initial_install_…

…options

Fixing initial install options

.gitignore

@@ -68,6 +68,7 @@ site
 config/proxy/acme.json
 .env
 secrets/live/*
+islandora_workbench
 
 #===================
 # Drupal and Mariadb files persistant storage

Makefile

@@ -18,7 +18,6 @@ endif
 # users to regenerate their .env files losing their changes.
 include sample.env
 include $(ENV_FILE)
-
 # The site to operate on when using drush -l $(SITE) commands
 SITE?=default
 
@@ -114,39 +113,40 @@ build:
 	docker build -f $(PROJECT_DRUPAL_DOCKERFILE) -t $(COMPOSE_PROJECT_NAME)_drupal --build-arg REPOSITORY=$(REPOSITORY) --build-arg TAG=$(TAG) .
 
 
-# Updates codebase folder to be owned by the host user and nginx group.
 .PHONY: set-files-owner
+## Updates codebase folder to be owned by the host user and nginx group.
 .SILENT: set-files-owner
 set-files-owner: $(SRC)
 ifndef SRC
 	$(error SRC is not set)
 endif
 	sudo find "$(SRC)" -exec chown $(shell id -u):101 {} \;
 
-# Creates required databases for drupal site(s) using environment variables.
 .PHONY: drupal-database
+## Creates required databases for drupal site(s) using environment variables.
 .SILENT: drupal-database
 drupal-database:
 	docker-compose exec -T drupal timeout 300 bash -c "while ! test -e /var/run/nginx/nginx.pid -a -e /var/run/php-fpm7/php-fpm7.pid; do sleep 1; done"
 	docker-compose exec -T drupal with-contenv bash -lc "for_all_sites create_database"
 
-# Installs drupal site(s) using environment variables.
 .PHONY: install
+## Installs drupal site(s) using environment variables.
 .SILENT: install
 install: drupal-database
 	docker-compose exec -T drupal with-contenv bash -lc "for_all_sites install_site"
 
-# Updates settings.php according to the environment variables.
 .PHONY: update-settings-php
+## Updates settings.php according to the environment variables.
 .SILENT: update-settings-php
 update-settings-php:
+	docker-compose exec -T drupal with-contenv bash -lc "if [ ! -f /var/www/drupal/web/sites/default/settings.php ]; then cp /var/www/drupal/web/sites/default/default.settings.php  /var/www/drupal/web/sites/default/settings.php; fi"
 	docker-compose exec -T drupal with-contenv bash -lc "for_all_sites update_settings_php"
 	# Make sure the host user can read the settings.php files after they have been updated.
 	sudo find ./codebase -type f -name "settings.php" -exec chown $(shell id -u):101 {} \;
 
-# Updates configuration from environment variables.
-# Allow all commands to fail as the user may not have all the modules like matomo, etc.
 .PHONY: update-config-from-environment
+## Updates configuration from environment variables.
+## Allow all commands to fail as the user may not have all the modules like matomo, etc.
 .SILENT: update-config-from-environment
 update-config-from-environment:
 	-docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_islandora_module"
@@ -157,20 +157,23 @@ update-config-from-environment:
 	-docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_openseadragon"
 	-docker-compose exec -T drupal with-contenv bash -lc "for_all_sites configure_islandora_default_module"
 
-# Runs migrations of islandora
 .PHONY: run-islandora-migrations
+## Runs migrations of islandora
 .SILENT: run-islandora-migrations
 run-islandora-migrations:
-	docker-compose exec -T drupal with-contenv bash -lc "for_all_sites import_islandora_migrations"
+	#docker-compose exec -T drupal with-contenv bash -lc "for_all_sites import_islandora_migrations"
+	# this line can be reverted when https://github.com/Islandora-Devops/isle-buildkit/blob/fae704f065435438828c568def2a0cc926cc4b6b/drupal/rootfs/etc/islandora/utilities.sh#L557
+	# has been updated to match
+	docker-compose exec -T drupal with-contenv bash -lc 'drush -l $(SITE) migrate:import islandora_defaults_tags,islandora_tags'
 
-# Creates solr-cores according to the environment variables.
 .PHONY: solr-cores
+## Creates solr-cores according to the environment variables.
 .SILENT: solr-cores
 solr-cores:
 	docker-compose exec -T drupal with-contenv bash -lc "for_all_sites create_solr_core_with_default_config"
 
-# Creates namespaces in Blazegraph according to the environment variables.
 .PHONY: namespaces
+## Creates namespaces in Blazegraph according to the environment variables.
 .SILENT: namespaces
 namespaces:
 	docker-compose exec -T drupal with-contenv bash -lc "for_all_sites create_blazegraph_namespace_with_default_properties"
@@ -208,7 +211,6 @@ remove_standard_profile_references_from_config:
 config-export:
 	docker-compose exec -T drupal drush -l $(SITE) config:export -y
 
-
 .PHONY: config-import
 .SILENT: config-import
 ## Import the sites configuration. N.B You may need to run this multiple times in succession due to errors in the configurations dependencies.
@@ -296,9 +298,8 @@ reindex-triplestore:
 	docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} vbo-exec content emit_node_event --configuration="queue=islandora-indexing-triplestore-index&event=Update"'
 	docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} vbo-exec media emit_media_event --configuration="queue=islandora-indexing-triplestore-index&event=Update"'
 
-# Helper to generate secrets & passwords, like so:
-# make generate-secrets
 .PHONY: generate-secrets
+## Helper to generate secrets & passwords, like so: make generate-secrets
 .SILENT: generate-secrets
 generate-secrets:
 ifeq ($(USE_SECRETS), false)
@@ -313,8 +314,8 @@ else
 	$(MAKE) secrets_warning
 endif
 
-# Helper function to generate keys for the user to use in their docker-compose.env.yml
 .PHONY: download-default-certs
+## Helper function to generate keys for the user to use in their docker-compose.env.yml
 .SILENT: download-default-certs
 download-default-certs:
 	mkdir -p certs
@@ -327,101 +328,67 @@ download-default-certs:
 
 .PHONY: demo
 .SILENT: demo
-## Make a demo site.
+## Make a local site from the install-profile and TODO then add demo content
 demo: generate-secrets
-	$(MAKE) download-default-certs ENVIROMENT=demo
-	$(MAKE) -B docker-compose.yml ENVIROMENT=demo
-	$(MAKE) pull ENVIROMENT=demo
-	mkdir -p "$(CURDIR)/codebase"
-	docker-compose up -d
-	$(MAKE) update-settings-php ENVIROMENT=demo
-	$(MAKE) drupal-public-files-import SRC="$(CURDIR)/build/demo-data/public-files.tgz" ENVIROMENT=demo
-	$(MAKE) drupal-database ENVIROMENT=demo
-	$(MAKE) drupal-database-import SRC="$(CURDIR)/build/demo-data/drupal.sql" ENVIROMENT=demo
-	$(MAKE) hydrate ENVIROMENT=demo
-	docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} upwd admin $${DRUPAL_DEFAULT_ACCOUNT_PASSWORD}'
-	$(MAKE) fcrepo-import SRC="$(CURDIR)/build/demo-data/fcrepo-export.tgz" ENVIROMENT=demo
-	$(MAKE) reindex-fcrepo-metadata ENVIROMENT=demo
-	$(MAKE) reindex-solr ENVIROMENT=demo
-	$(MAKE) reindex-triplestore ENVIROMENT=demo
-	$(MAKE) fix-masonry
-	$(MAKE) secrets_warning
+	$(MAKE) local
+	$(MAKE) demo_content
+	$(MAKE) login
 
 .PHONY: local
 .SILENT: local
-## Make a local site with codebase directory bind mounted.
+## Make a local site with codebase directory bind mounted, modeled after sandbox.islandora.ca
 local: QUOTED_CURDIR = "$(CURDIR)"
 local: generate-secrets
 	$(MAKE) download-default-certs ENVIROMENT=local
 	$(MAKE) -B docker-compose.yml ENVIRONMENT=local
 	$(MAKE) pull ENVIRONMENT=local
-	mkdir -p "$(CURDIR)/codebase"
-	if [ -z "$$(ls -A $(QUOTED_CURDIR)/codebase)" ]; then \
-		docker container run --rm -v "$(CURDIR)/codebase":/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'composer create-project drupal/recommended-project:^9.1 /tmp/codebase; mv /tmp/codebase/* /home/root; cd /home/root; composer config minimum-stability dev; composer require islandora/islandora:dev-8.x-1.x; composer require drush/drush:^10.3'; \
-	fi
-	docker-compose up -d
-	docker-compose exec -T drupal with-contenv bash -lc 'composer install; chown -R nginx:nginx .'
-	$(MAKE) remove_standard_profile_references_from_config ENVIROMENT=local
-	$(MAKE) install ENVIRONMENT=local
-	$(MAKE) hydrate ENVIRONMENT=local
-	$(MAKE) set-files-owner SRC="$(CURDIR)/codebase" ENVIROMENT=local
-	$(MAKE) secrets_warning
-
-.PHONY: demo-install-profile
-.SILENT: demo-instal-profile
-demo-install-profile: generate-secrets
-	$(MAKE) download-default-certs ENVIROMENT=demo
-	$(MAKE) -B docker-compose.yml ENVIROMENT=demo
-	$(MAKE) pull ENVIROMENT=demo
 	mkdir -p $(CURDIR)/codebase
-	docker-compose up -d --remove-orphans
-	@echo "\n Sleeping for 10 seconds to wait for Drupal to finish initializing.\n"
-	sleep 10
-	$(MAKE) install
-	$(MAKE) update-settings-php ENVIROMENT=demo
-	$(MAKE) hydrate ENVIROMENT=demo
-	docker-compose exec -T drupal with-contenv bash -lc 'drush --root /var/www/drupal/web -l $${DRUPAL_DEFAULT_SITE_URL} upwd admin $${DRUPAL_DEFAULT_ACCOUNT_PASSWORD}'
-	docker-compose exec -T drupal with-contenv bash -lc 'drush migrate:rollback islandora_defaults_tags,islandora_tags'
-	$(MAKE) initial_content
-	$(MAKE) login
-
-.PHONY: local-install-profile
-.SILENT: local-install-profile
-local-install-profile: generate-secrets
-	$(MAKE) download-default-certs ENVIROMENT=local
-	$(MAKE) -B docker-compose.yml ENVIRONMENT=local
-	$(MAKE) pull ENVIRONMENT=local
-	mkdir -p $(CURDIR)/codebase
-	if [ -z "$$(ls -A $(CURDIR)/codebase)" ]; then \
-		docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'git clone https://github.com/islandora-devops/islandora-sandbox /tmp/codebase; mv /tmp/codebase/* /home/root;'; \
+	if [ -z "$$(ls -A $(QUOTED_CURDIR)/codebase)" ]; then \
+		docker container run --rm -v $(CURDIR)/codebase:/home/root $(REPOSITORY)/nginx:$(TAG) with-contenv bash -lc 'git clone https://github.com/islandora-devops/islandora-sandbox -b main /tmp/codebase; mv /tmp/codebase/* /home/root;'; \
 	fi
 	$(MAKE) set-files-owner SRC=$(CURDIR)/codebase ENVIROMENT=local
 	docker-compose up -d --remove-orphans
 	docker-compose exec -T drupal with-contenv bash -lc 'composer install; chown -R nginx:nginx .'
-	$(MAKE) remove_standard_profile_references_from_config ENVIROMENT=local
-	$(MAKE) install ENVIRONMENT=local
+	$(MAKE) remove_standard_profile_references_from_config drupal-database update-settings-php ENVIROMENT=local
+	docker-compose exec -T drupal with-contenv bash -lc "drush si -y islandora_install_profile_demo --account-pass $(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD)"
+	$(MAKE) delete-shortcut-entities && docker-compose exec -T drupal with-contenv bash -lc "drush pm:un -y shortcut"
+	docker-compose exec -T drupal with-contenv bash -lc "drush en -y migrate_tools"
 	$(MAKE) hydrate ENVIRONMENT=local
-	# The - at the beginning is not a typo, it will allow this process to failing the make command.
 	-docker-compose exec -T drupal with-contenv bash -lc 'mkdir -p /var/www/drupal/config/sync && chmod -R 775 /var/www/drupal/config/sync'
-	docker-compose exec -T drupal with-contenv bash -lc 'chown -R `id -u`:101 /var/www/drupal'
-	docker-compose exec -T drupal with-contenv bash -lc 'drush migrate:rollback islandora_defaults_tags,islandora_tags'
-	$(MAKE) initial_content
+	#docker-compose exec -T drupal with-contenv bash -lc 'chown -R `id -u`:nginx /var/www/drupal'
+	#docker-compose exec -T drupal with-contenv bash -lc 'drush migrate:rollback islandora_defaults_tags,islandora_tags'
+	curl -k -u admin:$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) -H "Content-Type: application/json" -d "@build/demo-data/homepage.json" https://${DOMAIN}/node?_format=json
+	curl -k -u admin:$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) -H "Content-Type: application/json" -d "@build/demo-data/browse-collections.json" https://${DOMAIN}/node?_format=json
 	$(MAKE) login
 
-.PHONY: initial_content
-initial_content:
-	curl -u admin:$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) -H "Content-Type: application/json" -d "@demo-data/homepage.json" https://${DOMAIN}/node?_format=json
-	curl -u admin:$(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) -H "Content-Type: application/json" -d "@demo-data/browse-collections.json" https://${DOMAIN}/node?_format=json
+.PHONY: demo_content
+.SILENT: demo_content
+## Helper function for demo sites: do a workbench import of sample objects
+demo_content:
+	# fetch repo that has csv and binaries to data/samples
+	# if prod do this by default
+	# if [ -d "islandora_workbench" ]; then rm -rf islandora_workbench; fi
+	[ -d "islandora_workbench" ] || (git clone -b new_staging --single-branch https://github.com/DonRichards/islandora_workbench)
+ifeq ($(shell uname -s),Linux)
+	sed -i 's/^nopassword.*/password\: $(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) /g' islandora_workbench/demoBDcreate*
+	sed -i 's/http:/https:/g' islandora_workbench/demoBDcreate*
+endif
+ifeq ($(shell uname -s),Darwin)
+	sed -i '' 's/^nopassword.*/password\: $(shell cat secrets/live/DRUPAL_DEFAULT_ACCOUNT_PASSWORD) /g' islandora_workbench/demoBDcreate*
+	sed -i '' 's/http:/https:/g' islandora_workbench/demoBDcreate*
+endif
+	cd islandora_workbench && docker build -t workbench-docker .
+	cd islandora_workbench && docker run -it --rm --network="host" -v $(shell pwd)/islandora_workbench:/workbench --name my-running-workbench workbench-docker bash -lc "(cd /workbench && python setup.py install 2>&1 && ./workbench --config demoBDcreate_all_localhost.yml)"
+	$(MAKE) reindex-solr
 
-# Destroys everything beware!
 .PHONY: clean
 .SILENT: clean
 ## Destroys everything beware!
 clean:
 	echo "**DANGER** About to rm your SERVER data subdirs, your docker volumes and your codebase/web"
 	$(MAKE) confirm
 	-docker-compose down -v
-	sudo rm -fr codebase certs secrets/live/*
+	sudo rm -fr codebase islandora_workbench certs secrets/live/*
 	git clean -xffd .
 
 .PHONY: up
@@ -442,6 +409,7 @@ down:
 
 .PHONY: login
 .SILENT: login
+## Runs "drush uli" to provide a direct login link for user 1
 login:
 	echo "\n\n=========== LOGIN ==========="
 	docker-compose exec -T drupal with-contenv bash -lc "drush uli --uri=$(DOMAIN)"
@@ -475,19 +443,36 @@ TARGET_MAX_CHAR_NUM=20
 help:
 	@echo ''
 	@echo 'Usage:'
-	@echo '  ${RED}make${RESET} ${BLUE}<target>${RESET}'
+	@echo '  ${RED}make${RESET} ${BLUE}<function>${RESET}'
 	@echo ''
-	@echo 'Targets:'
+	@echo 'Functions to build:'
 	# @grep '^.PHONY: .* #' Makefile | sed 's/\.PHONY: \(.*\) # \(.*\)/\1 \2/'
 	@awk '/^[a-zA-Z\-\_0-9]+:/ { \
 		helpMessage = match(lastLine, /^## (.*)/); \
 		if (helpMessage) { \
 			helpCommand = $$1; sub(/:$$/, "", helpCommand); \
 			helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \
-			printf "  ${RED}%-$(TARGET_MAX_CHAR_NUM)s${RESET} ${BLUE}%s${RESET}\n", helpCommand, helpMessage; \
+			if (helpCommand == "up" || helpCommand == "local" || helpCommand == "demo") { \
+				printf "  ${RED}%-$(TARGET_MAX_CHAR_NUM)s${RESET} ${BLUE}%s${RESET}\n", helpCommand, helpMessage; \
+			} \
 		} \
 	} \
 	{lastLine = $$0}' $(MAKEFILE_LIST)
+	@echo ''
+	@echo 'Other functions:'
+	@awk '/^[a-zA-Z\-\_0-9]+:/ { \
+		helpMessage = match(lastLine, /^## (.*)/); \
+		if (helpMessage) { \
+			helpCommand = $$1; sub(/:$$/, "", helpCommand); \
+			helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \
+			# If helpCommand is not up, local, or demo then it is a target. \
+			if (helpCommand != "up" && helpCommand != "local" && helpCommand != "demo") { \
+				printf "  ${RED}%-$(TARGET_MAX_CHAR_NUM)s${RESET} ${BLUE}%s${RESET}\n", helpCommand, helpMessage; \
+			} \
+		} \
+	} \
+	{lastLine = $$0}' $(MAKEFILE_LIST)
+	@echo ''
 
 .PHONY: secrets_warning
 .SILENT: secrets_warning
@@ -513,10 +498,10 @@ set_admin_password:
 
 LATEST_VERSION := $(shell curl -s https://api.github.com/repos/desandro/masonry/releases/latest | grep '\"tag_name\":' | sed -E 's/.*\"([^\"]+)\".*/\1/')
 
-.PHONY: fix-masonry
-.SILENT: fix-masonry
+.PHONY: fix_masonry
+.SILENT: fix_masonry
 ## Fix missing masonry library.
-fix-masonry:
+fix_masonry:
 	@echo "Latest version of masonry library is ${LATEST_VERSION}"
 	docker-compose exec drupal bash -lc "[ -d '/var/www/drupal/web/libraries' ] && exit ; mkdir -p /var/www/drupal/web/libraries ; chmod 755 /var/www/drupal/web/libraries ; chown 1000:nginx /var/www/drupal/web/libraries"
 	docker-compose exec drupal bash -lc "cd /var/www/drupal/web/libraries/ ; [ ! -d '/var/www/drupal/web/libraries/masonry' ] && git clone --quiet --branch ${LATEST_VERSION} https://github.com/desandro/masonry.git || echo Ready"

README.md

@@ -23,7 +23,7 @@
 ## Introduction
 
 [Docker Compose] project for creating and managing an Islandora 8 instance
-using [Docker] containers from [Docker Hub](https://hub.docker.com/u/islandora) 
+using [Docker] containers from [Docker Hub](https://hub.docker.com/u/islandora)
 that were created by [isle-buildkit](https://github.com/Islandora-Devops/isle-buildkit).
 
 In a nutshell, `isle-dc` generates a docker-compose.yml file for you based on configuration
@@ -40,7 +40,7 @@ as database import/export and reindexing.
 
 - Desktop / laptop / VM (*Docker must have sufficient resources to run GNU Make*)
 - Docker-CE 19.x+
-- Docker-compose version 1.25.x+ 
+- Docker-compose version 1.25.x+
 - Git 2.0+
 - GNU Make 4.0+
 - At least 8GB of RAM (ideally 16GB)
@@ -58,6 +58,9 @@ To get started with a **demo** environment, run:
 ```bash
 make demo
 ```
+
+Or if this demo environment is too simple, run `make demo` to create a demo site that is modeled after https://sandbox.islandora.ca with some extra content imported via Islandora Workbench.
+
 ⚠️ If prompted during `make up\demo\local\clean` for password, use your computer's password. The build process may need elevated privileges to write or remove files. For other password information see [Secrets](#secrets)
 
 This will pull down images from Dockerhub and generate
@@ -97,7 +100,7 @@ When you're done with your demo environment, shut it down by running
 docker-compose down
 ```
 
-This will keep your data around until the next time you start your instance.  If you want to completely destroy the repository and 
+This will keep your data around until the next time you start your instance.  If you want to completely destroy the repository and
 all ingested data, use
 
 ```
@@ -129,12 +132,14 @@ INSTALL_EXISTING_CONFIG=true
 DRUPAL_INSTALL_PROFILE=minimal
 ```
 
-In either case, run this command to make a local environment.
+In either case, run one of these commands to make a local environment.
 
 ```bash
 make local
 ```
 
+The former will create a starter site modeled off of https://sandbox.islandora.ca.
+
 If you already have a Drupal site but don't know how to export it,
 log into your server, navigate to the Drupal root, and run the following commands:
 
@@ -186,7 +191,7 @@ make down
 # Bring isle-dc back up from where it left off
 make up
 
-# If make hasn't been run this will run make demo 
+# If make hasn't been run this will run make demo
 
 ```
 
@@ -320,7 +325,7 @@ INCLUDE_WATCHTOWER_SERVICE=true
 ### Traefik
 
 The [traefik](https://containo.us/traefik/) container acts as a reverse proxy,
-and exposes some containers through port ``80``/``443``/``3306``. 
+and exposes some containers through port ``80``/``443``/``3306``.
 
 Since Drupal passes links to itself in the messages it passes to the microservices,
 and occasionally other URLs need to be resolved on containers that do not have