Merge remote-tracking branch 'upstream/master' into setup-integration-tests-m24

.github/workflows/shellcheck.yml

@@ -0,0 +1,24 @@
+name: 'ShellCheck'
+
+on: 
+  push:
+    branches: ["master"]
+  pull_request:
+
+jobs:
+  shellcheck:
+    name: shellcheck
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Run ShellCheck
+      uses: ludeeus/action-shellcheck@master
+      id: check
+      env:
+        SHELLCHECK_OPTS: -x -e SC2181 -P compose/bin -P compose/env
+      with:
+        check_together: true
+        scandir: './compose/bin'

CHANGELOG.md

@@ -6,7 +6,39 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
-- New PHP image with tag `8.0-fpm-develop` available for testing purposes.
+- PHP image `8.0-fpm-develop` now available for testing purposes.
+
+## [36.0.1] - 2021-02-04
+
+### Updated
+- Reverted DB image back to `percona:5.7` until issues with MySQL 8.0 image are resolved.
+
+## [36.0.0] - 2021-02-04
+
+### Added
+- New Elasticsearch Docker images `7.7`, `7.7.1-0` [#392](https://github.com/markshust/docker-magento/issues/392).
+- SpellCheck GitHub Action for continuous integration checks of shell scripts [#387](https://github.com/markshust/docker-magento/pull/387), [#388](https://github.com/markshust/docker-magento/pull/388).
+
+### Fixed
+- Support filesystem paths with spaces [e5f22e56](https://github.com/markshust/docker-magento/commit/e5f22e56fcd382b8339d5804a9d236dd6b238a3d).
+- Added missing `bin/cache-clean` file [f0e57202](https://github.com/markshust/docker-magento/commit/f0e5720281cd9f536f163bd5bdfe5bd66a956dc6).
+
+### Updated
+- Updated PHP images to NodeJS version 14 LTS [4a81f2b8](https://github.com/markshust/docker-magento/commit/4a81f2b8c61674b261ee7b42752e21fc8d5e945d).
+- Changed `db` service to use MySQL 8.0 Docker image.
+
+## [35.0.0] - 2021-01-29
+
+### Added
+- Automatically purge caches for a better dev experience [#380](https://github.com/markshust/docker-magento/issues/380).
+- Stop script execution on error [#363](https://github.com/markshust/docker-magento/pull/363/).
+- Make xdebug command understand partials [#371](https://github.com/markshust/docker-magento/pull/371).
+- Extended functionality for `bin/xdebug`, including new `status` and `toggle` commands [#332](https://github.com/markshust/docker-magento/pull/332).
+- Check Elasticsearch connection before setup:install [#326](https://github.com/markshust/docker-magento/pull/326).
+
+### Updated
+- The onelinesetup now accepts a `community` or `enterprise` param to pick version to install [b2399ff1](https://github.com/markshust/docker-magento/commit/ad573f6f3c8d2f7066034cbde936a86eb2399ff1).
+- Fix bin/start for macOS Big Sur [#355](https://github.com/markshust/docker-magento/pull/355/).
 
 ## [34.2.0] - 2020-10-15
 

README.md

@@ -89,14 +89,16 @@ View Dockerfiles:
       - [`1.18-1`](https://github.com/markshust/docker-magento/tree/31.0.1/images/nginx/1.18)
       - [`1.18-0`](https://github.com/markshust/docker-magento/tree/31.0.0/images/nginx/1.18)
 - [markoshust/magento-php (Docker Hub)](https://hub.docker.com/r/markoshust/magento-php/)
-  - 8.0 (Currently in "development" for testing purposes.)
+  - 8.0 (available for alpha testing)
       - [`8.0-fpm-develop`](https://github.com/markshust/docker-magento/tree/master/images/php/8.0)
   - 7.4
-      - [`7.4-fpm`, `7.4-fpm-2`](https://github.com/markshust/docker-magento/tree/master/images/php/7.4)
+      - [`7.4-fpm`, `7.4-fpm-3`](https://github.com/markshust/docker-magento/tree/master/images/php/7.4)
+      - [`7.4-fpm-2`](https://github.com/markshust/docker-magento/tree/34.2.0/images/php/7.4)
       - [`7.4-fpm-1`](https://github.com/markshust/docker-magento/tree/34.1.0/images/php/7.4)
       - [`7.4-fpm-0`](https://github.com/markshust/docker-magento/tree/33.0.0/images/php/7.4)
   - 7.3
-      - [`7.3-fpm`, `7.3-fpm-9`](https://github.com/markshust/docker-magento/tree/master/images/php/7.3)
+      - [`7.3-fpm`, `7.3-fpm-10`](https://github.com/markshust/docker-magento/tree/master/images/php/7.3)
+      - [`7.3-fpm-9`](https://github.com/markshust/docker-magento/tree/34.2.0/images/php/7.3)
       - [`7.3-fpm-8`](https://github.com/markshust/docker-magento/tree/34.1.0/images/php/7.3)
       - [`7.3-fpm-7`](https://github.com/markshust/docker-magento/tree/33.0.0/images/php/7.3)
       - [`7.3-fpm-6`](https://github.com/markshust/docker-magento/tree/32.0.1/images/php/7.3)
@@ -108,7 +110,8 @@ View Dockerfiles:
       - [`7.3-fpm-0`](https://github.com/markshust/docker-magento/tree/24.2.0/images/php/7.3)
 - [markoshust/magento-elasticsearch (Docker Hub)](https://hub.docker.com/r/markoshust/magento-elasticsearch/)
   - 7
-      - [`7.6`, `7.6.2-2`](https://github.com/markshust/docker-magento/tree/master/images/elasticsearch/7.6)
+      - [`7.7`, `7.7.1-0`](https://github.com/markshust/docker-magento/tree/master/images/elasticsearch/7.7)
+      - [`7.6`, `7.6.2-2`](https://github.com/markshust/docker-magento/tree/35.0.0/images/elasticsearch/7.6)
       - [`7.6.2-1`](https://github.com/markshust/docker-magento/tree/32.0.0/images/elasticsearch/7.6)
       - [`7.6.2-0`](https://github.com/markshust/docker-magento/tree/31.0.2/images/elasticsearch/7.6)
 
@@ -202,7 +205,7 @@ cp -R ~/Sites/existing src
 # Create a DNS host entry for the site:
 echo "127.0.0.1 ::1 yoursite.test" | sudo tee -a /etc/hosts
 
-# Start some containers, copy files ot them and then restart the containers:
+# Start some containers, copy files to them and then restart the containers:
 docker-compose up -d
 rm -rf src/vendor
 bin/copytocontainer --all ## Initial copy will take a few minutes...
@@ -247,21 +250,24 @@ It is recommended to keep your root docker config files in one repository, and y
 ## Custom CLI Commands
 
 - `bin/bash`: Drop into the bash prompt of your Docker container. The `phpfpm` container should be mainly used to access the filesystem within Docker.
+- `bin/cache-clean`: Access the [cache-clean](https://github.com/mage2tv/magento-cache-clean) CLI. Note the watcher is automatically started at startup in `bin/start`. Ex. `bin/cache-clean config full_page`
 - `bin/cli`: Run any CLI command without going into the bash prompt. Ex. `bin/cli ls`
 - `bin/clinotty`: Run any CLI command with no TTY. Ex. `bin/clinotty chmod u+x bin/magento`
+- `bin/cliq`: The same as `bin/cli`, but pipes all output to `/dev/null`. Useful for a quiet CLI, or implementing long-running processes.
 - `bin/composer`: Run the composer binary. Ex. `bin/composer install`
 - `bin/copyfromcontainer`: Copy folders or files from container to host. Ex. `bin/copyfromcontainer vendor`
 - `bin/copytocontainer`: Copy folders or files from host to container. Ex. `bin/copytocontainer --all`
 - `bin/dev-urn-catalog-generate`: Generate URN's for PHPStorm and remap paths to local host. Restart PHPStorm after running this command.
 - `bin/devconsole`: Alias for `bin/n98-magerun2 dev:console`
-- `bin/download`: Download & extract specific Magento version to the `src` directory. If the archive download fails, it will attempt to download with Composer. Ex. `bin/download 2.4.1`.
+- `bin/devtools-cli-check`: Check & install the CLI devtools if missing from system.
+- `bin/download`: Download & extract specific Magento version to the `src` directory. If the archive download fails, it will attempt to download with Composer. Ex. `bin/download 2.4.1`
 - `bin/fixowns`: This will fix filesystem ownerships within the container.
 - `bin/fixperms`: This will fix filesystem permissions within the container.
 - `bin/grunt`: Run the grunt binary. Ex. `bin/grunt exec`
 - `bin/magento`: Run the Magento CLI. Ex: `bin/magento cache:flush`
 - `bin/mysql`: Run the MySQL CLI with database config from `env/db.env`. Ex. `bin/mysql -e "EXPLAIN core_config_data"` or`bin/mysql < backups/magento.sql`
 - `bin/mysqldump`: Backup the Magento database. Ex. `bin/mysqldump > backups/magento.sql`
-- `bin/n98-magerun2`: Access the n98 magerun CLI. Ex: `bin/n98-magerun2 dev:console`
+- `bin/n98-magerun2`: Access the [n98-magerun2](https://github.com/netz98/n98-magerun2) CLI. Ex: `bin/n98-magerun2 dev:console`
 - `bin/node`: Run the node binary. Ex. `bin/node --version`
 - `bin/npm`: Run the npm binary. Ex. `bin/npm install`
 - `bin/pwa-studio`: (BETA) Start the PWA Studio server. Note that Chrome will throw SSL cert errors and not allow you to view the site, but Firefox will.
@@ -285,6 +291,12 @@ It is recommended to keep your root docker config files in one repository, and y
 
 ## Misc Info
 
+### Caching
+
+For an improved developer experience, caches are automatically refreshed when related files are updated, courtesy of [cache-clean](https://github.com/mage2tv/magento-cache-clean). This means you can keep all of the standard Magento caches enabled, and this script will only clear the specific caches needed, and only when necessary.
+
+To disable this functionality, uncomment the last line in the `bin/start` file to disable the watcher.
+
 ### Database
 
 The hostname of each service is the name of the service within the `docker-compose.yml` file. So for example, MySQL's hostname is `db` (not `localhost`) when accessing it from within a Docker container. Elasticsearch's hostname is `elasticsearch`.
@@ -364,7 +376,7 @@ Otherwise, this project now automatically sets up Xdebug support with VS Code. I
         * Remove any pre-existing volume bindings.
         * Ensure a volume binding has been setup for Container path of `/var/www/html` mapped to the Host path of `./src`.
 
-4. Open `PHPStorm > Preferences > Languages & Frameworks > PHP > Debug` and set Debug Port to `9001`.
+4. Open `PHPStorm > Preferences > Languages & Frameworks > PHP > Debug` and set Debug Port to `9001,9003`.
 
 5. Open `PHPStorm > Preferences > Languages & Frameworks > PHP > DBGp Proxy` and set Port to `9001`.
 
@@ -382,7 +394,9 @@ Otherwise, this project now automatically sets up Xdebug support with VS Code. I
 
 Running Docker on Linux should be pretty straight-forward. Note that you need to run some [post install commands](https://docs.docker.com/install/linux/linux-postinstall/) as well as [installing Docker Compose](https://docs.docker.com/compose/install/). These steps are taken care of automatically with Docker Desktop, but not on Linux.
 
-You may have to increase a virtual memory map count on the host system. It is required by [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html).
+Be sure to see the "Linux only" documentation in the [docker-compose.dev.yml](https://github.com/markshust/docker-magento/blob/master/compose/docker-compose.dev.yml#L30) file. The `extra_hosts` param is required to be defined on Linux for proper DNS resolution.
+
+You may also have to increase a virtual memory map count on the host system. It is required by [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html).
 
 Add following line to `/etc/sysctl.conf`:
 
@@ -390,6 +404,12 @@ Add following line to `/etc/sysctl.conf`:
 vm.max_map_count=262144
 ```
 
+To enable Xdebug on linux, you'll also need to open port 9001 on the firewall with:
+
+```
+sudo iptables -A INPUT -p tcp --dport 9001 -j ACCEPT
+```
+
 ### Blackfire.io
 
 These docker images have built-in support for Blackfire.io. To use it, first register your server ID and token with the Blackfire agent:

compose/bin/cache-clean

@@ -0,0 +1,2 @@
+#!/bin/bash
+bin/cli /var/www/.composer-global/vendor/bin/cache-clean.js "$@"

compose/bin/cliq

@@ -0,0 +1,3 @@
+#!/bin/bash
+[ -z "$1" ] && echo "Please specify a CLI command (ex. ls)" && exit
+bin/clinotty "$@" >/dev/null

compose/bin/copyfromcontainer

@@ -3,13 +3,13 @@
 
 REAL_SRC=$(cd -P "src" && pwd)
 if [ "$1" == "--all" ]; then
-  docker cp $(docker-compose ps -q phpfpm|awk '{print $1}'):/var/www/html/./ $REAL_SRC/
+  docker cp "$(docker-compose ps -q phpfpm|awk '{print $1}')":/var/www/html/./ "$REAL_SRC/"
   echo "Completed copying all files from container to host"
 else
   if [ -f "$1" ] ; then
-    docker cp $(docker-compose ps -q phpfpm|awk '{print $1}'):/var/www/html/$1 $REAL_SRC/$1
+    docker cp "$(docker-compose ps -q phpfpm|awk '{print $1}')":/var/www/html/"$1" "$REAL_SRC/$1"
   else
-    docker cp $(docker-compose ps -q phpfpm|awk '{print $1}'):/var/www/html/$1 $REAL_SRC/$(dirname $1)
+    docker cp "$(docker-compose ps -q phpfpm|awk '{print $1}')":/var/www/html/"$1" "$REAL_SRC/$(dirname "$1")"
   fi
   echo "Completed copying $1 from container to host"
 fi

compose/bin/copytocontainer

@@ -3,17 +3,17 @@
 
 REAL_SRC=$(cd -P "src" && pwd)
 if [ "$1" == "--all" ]; then
-  docker cp $REAL_SRC/./ $(docker-compose ps -q phpfpm|awk '{print $1}'):/var/www/html/
+  docker cp "$REAL_SRC/./" "$(docker-compose ps -q phpfpm|awk '{print $1}')":/var/www/html/
   echo "Completed copying all files from host to container"
   bin/fixowns
   bin/fixperms
 else
   if [ -f "$REAL_SRC/$1" ]; then
-    docker cp $REAL_SRC/$1 $(docker-compose ps -q phpfpm|awk '{print $1}'):/var/www/html/$1
+    docker cp "$REAL_SRC/${1}" "$(docker-compose ps -q phpfpm|awk '{print $1}')":/var/www/html/"$1"
   else
-    docker cp $REAL_SRC/$1 $(docker-compose ps -q phpfpm|awk '{print $1}'):/var/www/html/$(dirname $1)
+    docker cp "$REAL_SRC/${1}" "$(docker-compose ps -q phpfpm|awk '{print $1}')":/var/www/html/"$(dirname "$1")"
   fi
   echo "Completed copying $1 from host to container"
-  bin/fixowns $1
-  bin/fixperms $1
+  bin/fixowns "$1"
+  bin/fixperms "$1"
 fi

compose/bin/devtools-cli-check

@@ -0,0 +1,8 @@
+#!/bin/bash
+if ! bin/clinotty ls /var/www/.composer-global/vendor/bin/cache-clean.js 1> /dev/null 2>&1; then
+  echo "Installing devtools metapackage, just a moment..."
+  bin/cliq mkdir -p /var/www/.composer-global
+  bin/composer init --working-dir=/var/www/.composer-global --quiet --no-interaction
+  bin/composer require --working-dir=/var/www/.composer-global --quiet markshust/magento2-metapackage-devtools-cli:^1.0
+  echo "Devtools installed."
+fi

compose/bin/download

@@ -1,31 +1,32 @@
 #!/bin/bash
+set -o errexit
 [ -z "$1" ] && echo "Please specify the version to download (ex. 2.0.0)" && exit
 
 edition=${2:-community}
 
 if [[ "$edition" == "enterprise" ]]; then
     rm -rf src
-    composer create-project --repository=https://repo.magento.com/ --ignore-platform-reqs magento/project-enterprise-edition=$1 src
+    composer create-project --repository=https://repo.magento.com/ --ignore-platform-reqs magento/project-enterprise-edition="$1" src
     exit 0
 fi
 
-if [ ! -f ~/.docker-magento/magento2-$1.tar.gz ]; then
+if [ ! -f ~/.docker-magento/magento2-"$1".tar.gz ]; then
     mkdir -p ~/.docker-magento
-    (cd ~/.docker-magento && curl -fOL http://pubfiles.nexcess.net/magento/ce-packages/magento2-$1.tar.gz)
+    (cd ~/.docker-magento && curl -fOL http://pubfiles.nexcess.net/magento/ce-packages/magento2-"$1".tar.gz)
 fi
 
 # Fallback download to hypernode if archive doesn't exist on Nexcess (smaller than 1MB)
-if [ $(find ~/.docker-magento/magento2-$1.tar.gz -size -1M) ]; then
-    (cd ~/.docker-magento && curl -o magento2-$1.tar.gz -fOL https://www.magento.mirror.hypernode.com/releases/magento-$1.tar.gz)
+if [ "$(find ~/.docker-magento/magento2-"$1".tar.gz -size -1M)" ]; then
+    (cd ~/.docker-magento && curl -o magento2-"$1".tar.gz -fOL https://www.magento.mirror.hypernode.com/releases/magento-"$1".tar.gz)
 fi
 
 # Final fallback. If no archive exists, let's use Composer!
-if [ ! -f ~/.docker-magento/magento2-$1.tar.gz ]; then
+if [ ! -f ~/.docker-magento/magento2-"$1".tar.gz ]; then
     echo "Archive not found, or not yet available due to new version release."
     echo "Attempting install with Composer..."
     rm -rf src
-    composer create-project --repository=https://repo.magento.com/ --ignore-platform-reqs magento/project-community-edition=$1 src
+    composer create-project --repository=https://repo.magento.com/ --ignore-platform-reqs magento/project-community-edition="$1" src
 else
     echo "Extracting magento2-$1.tar.gz to ./src"
-    mkdir -p src && tar xzf ~/.docker-magento/magento2-$1.tar.gz -o -C src
+    mkdir -p src && tar xzf ~/.docker-magento/magento2-"$1".tar.gz -o -C src
 fi

compose/bin/fixowns

@@ -4,7 +4,7 @@ echo "Correcting filesystem ownerships..."
 if [ -z "$1" ]; then
   bin/rootnotty chown -R app:app /var/www/
 else
-  bin/rootnotty chown -R app:app /var/www/html/$1
+  bin/rootnotty chown -R app:app /var/www/html/"$1"
 fi
 
 echo "Filesystem ownerships corrected."

compose/bin/mysql

@@ -1,9 +1,11 @@
 #!/bin/bash
+
+# shellcheck source=../env/db.env
 source env/db.env
 if [ -t 0 ]; then
   # Need tty to run mysql shell
-  bin/cli mysql -h${MYSQL_HOST} -u${MYSQL_USER} -p${MYSQL_PASSWORD} ${MYSQL_DATABASE} "$@"
+  bin/cli mysql -h"${MYSQL_HOST}" -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" "${MYSQL_DATABASE}" "$@"
 else
   # Read from stdin, ex: bin/mysql < dbdump.sql
-  bin/clinotty mysql -h${MYSQL_HOST} -u${MYSQL_USER} -p${MYSQL_PASSWORD} ${MYSQL_DATABASE} "$@"
+  bin/clinotty mysql -h"${MYSQL_HOST}" -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" "${MYSQL_DATABASE}" "$@"
 fi

compose/bin/pwa-studio

@@ -1,4 +1,6 @@
 #!/bin/bash
+set -o errexit
+
 if [ ! -d pwa-studio ]; then
     echo "PWA studio must first be installed by running bin/setup-pwa-studio"
     exit

compose/bin/removevolumes

@@ -1,8 +1,8 @@
 #!/bin/bash
 current_folder=${PWD##*/}
-volume_prefix=`echo $current_folder | awk '{print tolower($0)}' | sed 's/\.//g'`
-docker volume rm ${volume_prefix}_appdata
-docker volume rm ${volume_prefix}_dbdata
-docker volume rm ${volume_prefix}_rabbitmqdata
-docker volume rm ${volume_prefix}_sockdata
-docker volume rm ${volume_prefix}_ssldata
+volume_prefix=$(echo "$current_folder" | awk '{print tolower($0)}' | sed 's/\.//g')
+docker volume rm "${volume_prefix}"_appdata
+docker volume rm "${volume_prefix}"_dbdata
+docker volume rm "${volume_prefix}"_rabbitmqdata
+docker volume rm "${volume_prefix}"_sockdata
+docker volume rm "${volume_prefix}"_ssldata

compose/bin/setup

@@ -1,7 +1,13 @@
 #!/bin/bash
+set -o errexit
+
+# shellcheck source=../env/db.env
 source env/db.env
 BASE_URL=${1:-magento2.test}
 
+ES_HOST=elasticsearch
+ES_PORT=9200
+
 bin/stop
 
 docker-compose -f docker-compose.yml up -d
@@ -20,13 +26,21 @@ echo "Forcing reinstall of composer deps to ensure perms & reqs..."
 bin/clinotty composer global require hirak/prestissimo
 bin/clinotty composer update
 
+echo "Waiting for connection to Elasticsearch..."
+bin/clinotty timeout 30 bash -c "
+    until curl --silent --output /dev/null http://$ES_HOST:$ES_PORT/_cat/health?h=st; do
+        printf '.'
+        sleep 5
+    done"
+[ $? != 0 ] && echo "Failed to connect to Elasticsearch" && exit
+
 bin/clinotty bin/magento setup:install \
-  --db-host=$MYSQL_HOST \
-  --db-name=$MYSQL_DATABASE \
-  --db-user=$MYSQL_USER \
-  --db-password=$MYSQL_PASSWORD \
-  --base-url=https://$BASE_URL/ \
-  --base-url-secure=https://$BASE_URL/ \
+  --db-host="$MYSQL_HOST" \
+  --db-name="$MYSQL_DATABASE" \
+  --db-user="$MYSQL_USER" \
+  --db-password="$MYSQL_PASSWORD" \
+  --base-url=https://"$BASE_URL"/ \
+  --base-url-secure=https://"$BASE_URL"/ \
   --backend-frontname=admin \
   --admin-firstname=John \
   --admin-lastname=Smith \
@@ -52,7 +66,8 @@ bin/clinotty bin/magento setup:install \
   --session-save-redis-log-level=4 \
   --session-save-redis-db=2 \
   --search-engine=elasticsearch7 \
-  --elasticsearch-host=elasticsearch \
+  --elasticsearch-host=$ES_HOST \
+  --elasticsearch-port=$ES_PORT \
   --use-rewrites=1
 
 echo "Turning on developer mode.."
@@ -74,7 +89,7 @@ bin/copyfromcontainer app
 bin/copyfromcontainer vendor
 
 echo "Generating SSL certificate..."
-bin/setup-ssl $BASE_URL
+bin/setup-ssl "$BASE_URL"
 
 echo "Docker development environment setup complete."
 echo "You may now access your Magento instance at https://${BASE_URL}/"

compose/bin/setup-composer-auth

@@ -27,14 +27,14 @@ elif [ $USE_PYTHON3 ]; then
     PY3_PASS="import sys, json; print(json.load(sys.stdin)['http-basic']['repo.magento.com']['password'])"
 
     if [ -f "$GLOBAL_AUTH" ]; then
-        PUBLIC_KEY=$(cat "$GLOBAL_AUTH" | python3 -c "$PY3_USER" 2>/dev/null)
-        PRIVATE_KEY=$(cat "$GLOBAL_AUTH" | python3 -c "$PY3_USER" 2>/dev/null)
+        PUBLIC_KEY=$(python3 -c "$PY3_USER" 2>/dev/null < "$GLOBAL_AUTH")
+        PRIVATE_KEY=$(python3 -c "$PY3_USER" 2>/dev/null < "$GLOBAL_AUTH")
     fi
 
     if [ -z "$PUBLIC_KEY" ] || [ -z "$PRIVATE_KEY" ]; then
         if [ -f "$PROJECT_AUTH" ]; then
-            PUBLIC_KEY=$(cat "$PROJECT_AUTH" | python3 -c "$PY3_USER" 2>/dev/null)
-            PRIVATE_KEY=$(cat "$PROJECT_AUTH" | python3 -c "$PY3_PASS" 2>/dev/null)
+            PUBLIC_KEY=$(python3 -c "$PY3_USER" 2>/dev/null < "$PROJECT_AUTH")
+            PRIVATE_KEY=$(python3 -c "$PY3_PASS" 2>/dev/null < "$PROJECT_AUTH")
             NEED_AUTH=true
         fi
     fi
@@ -49,8 +49,8 @@ if [ -z "$PUBLIC_KEY" ] || [ -z "$PRIVATE_KEY" ]; then
     exec < /dev/tty
     echo
     echo "    Authentication required (repo.magento.com, public_key and private_key):"
-    read -p "        Username: " PUBLIC_KEY
-    read -p "        Password: " PRIVATE_KEY
+    read -r -p "        Username: " PUBLIC_KEY
+    read -r -p "        Password: " PRIVATE_KEY
     echo
     exec <&-
 fi
@@ -60,9 +60,9 @@ if [ -z "$PUBLIC_KEY" ] || [ -z "$PRIVATE_KEY" ]; then
 fi
 
 # For docker-compose.yml setting: ~/.composer:/var/www/.composer:cached
-echo "Authentication will add to host composer global config ($GLOBAL_AUTH) for docekr container"
+echo "Authentication will add to host composer global config ($GLOBAL_AUTH) for docker container"
 if [ $USE_COMPOSER ]; then
-    composer global config http-basic.repo.magento.com $PUBLIC_KEY $PRIVATE_KEY
+    composer global config http-basic.repo.magento.com "$PUBLIC_KEY" "$PRIVATE_KEY"
 elif [ $USE_PYTHON3 ]; then
     PY3_MERGE_AUTH="""
 import sys, json;
@@ -89,11 +89,11 @@ def merge(src, dest):
 print(json.dumps(merge(auth, data), indent=4))
 """
     if [ -f "$GLOBAL_AUTH" ]; then
-        mkdir -p $(dirname "$GLOBAL_AUTH")
+        mkdir -p "$(dirname "$GLOBAL_AUTH")"
         echo "{}" > "$GLOBAL_AUTH"
     fi
     mv "$GLOBAL_AUTH" "$GLOBAL_AUTH.bak"
-    cat "$GLOBAL_AUTH.bak" | python3 -c "$PY3_MERGE_AUTH" > "$GLOBAL_AUTH"
+    python3 -c "$PY3_MERGE_AUTH" > "$GLOBAL_AUTH" < "$GLOBAL_AUTH.bak"
 fi
 
 echo "Success to setup composer auth"

compose/bin/setup-grunt

@@ -1,11 +1,11 @@
 #!/bin/bash
 DEFAULT_THEME_ID="select value from core_config_data where path = 'design/theme/theme_id'"
 THEME_PATH="select theme_path from theme where theme_id in ($DEFAULT_THEME_ID);"
-VENDOR_THEME=`bin/n98-magerun2 db:query "$THEME_PATH" | sed -n 2p | cut -d$'\r' -f1`
-THEME=`echo $VENDOR_THEME | cut -d'/' -f2`
+VENDOR_THEME=$(bin/n98-magerun2 db:query "$THEME_PATH" | sed -n 2p | cut -d$'\r' -f1)
+THEME=$(echo "$VENDOR_THEME" | cut -d'/' -f2)
 
 # Generate local-theme.js for custom theme
-read -d '' GEN_THEME_JS << EOM
+read -r -d '' GEN_THEME_JS << EOM
 var fs = require('fs');
 var util = require('util');
 var theme = require('./dev/tools/grunt/configs/themes');

compose/bin/setup-integration-tests

@@ -1,18 +1,20 @@
 #!/bin/bash
+
+# shellcheck source=../env/db.env
 source env/db.env
 
 MYSQL_INTEGRATION_CONFIG=dev/tests/integration/etc/install-config-mysql.php
 
 # If database doesn't exist create it and add user permissions
-bin/clinotty mysql -h${MYSQL_INTEGRATION_HOST} -uroot -p${MYSQL_ROOT_PASSWORD} ${MYSQL_INTEGRATION_DATABASE} -e exit &> /dev/null ||
-  bin/clinotty mysqladmin -h${MYSQL_INTEGRATION_HOST} -uroot -p${MYSQL_ROOT_PASSWORD} create ${MYSQL_INTEGRATION_DATABASE} &&
+bin/clinotty mysql -h"${MYSQL_INTEGRATION_HOST}" -uroot -p"${MYSQL_ROOT_PASSWORD}" "${MYSQL_INTEGRATION_DATABASE}" -e exit &> /dev/null ||
+  bin/clinotty mysqladmin -h"${MYSQL_INTEGRATION_HOST}" -uroot -p"${MYSQL_ROOT_PASSWORD}" create "${MYSQL_INTEGRATION_DATABASE}" &&
   echo "Database ${MYSQL_INTEGRATION_DATABASE} created." &&
-  bin/cli mysql -uroot -p${MYSQL_ROOT_PASSWORD} -h${MYSQL_INTEGRATION_HOST} \
+  bin/cli mysql -uroot -p"${MYSQL_ROOT_PASSWORD}" -h"${MYSQL_INTEGRATION_HOST}" \
     -e "GRANT ALL PRIVILEGES ON ${MYSQL_INTEGRATION_DATABASE}.* TO '${MYSQL_INTEGRATION_USER}'@'%';FLUSH PRIVILEGES;"
 
 if [[ ! -f "src/${MYSQL_INTEGRATION_CONFIG}" ]]; then
   MAGENTO_VERSION=$(bin/magento --version --no-ansi | cut -d" " -f 3)
-  MAGENTO_VERSION_SEGMENTS=(${MAGENTO_VERSION//./ })
+  IFS=. read -r -a MAGENTO_VERSION_SEGMENTS <<< "${MAGENTO_VERSION}"
   MAGENTO_MAJOR="${MAGENTO_VERSION_SEGMENTS[0]}.${MAGENTO_VERSION_SEGMENTS[1]}"
   cp template/"${MYSQL_INTEGRATION_CONFIG}"."${MAGENTO_MAJOR}".dist src/${MYSQL_INTEGRATION_CONFIG}
 fi

compose/bin/setup-pwa-studio

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -o errexit
 echo "Install NodeJS and Yarn on host machine, otherwise setup will fail"
 
 BASE_URL=${1:-master-7rqtwti-mfwmkrjfqvbjk.us-4.magentosite.cloud}

compose/bin/setup-ssl-ca

@@ -1,9 +1,9 @@
 #!/bin/bash
-
+set -o errexit
 # Generate a new local CA "/root/.local/share/mkcert"
 docker-compose exec -T -u root app mkcert -install
 
-docker cp $(docker-compose ps -q app|awk '{print $1}'):/root/.local/share/mkcert/rootCA.pem .
+docker cp "$(docker-compose ps -q app|awk '{print $1}')":/root/.local/share/mkcert/rootCA.pem .
 echo "System password requested to install certificate authority on host..."
 
 if [ "$(uname)" == "Darwin" ]; then

compose/bin/start

@@ -1,25 +1,31 @@
 #!/bin/bash
+set -o errexit
+
+# Ref: https://stackoverflow.com/a/51789677/9821321
 function parseYaml {
-  local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
+  local s
+  local w
+  local fs
+  s='[[:space:]]*'
+  w='[a-zA-Z0-9_]*'
+  fs=$(echo @|tr @ '\034')
   sed -ne "s|,$s\]$s\$|]|" \
-      -e ":1;s|^\($s\)\($w\)$s:$s\[$s\(.*\)$s,$s\(.*\)$s\]|\1\2: [\3]\n\1  - \4|;t1" \
-      -e "s|^\($s\)\($w\)$s:$s\[$s\(.*\)$s\]|\1\2:\n\1  - \3|;p" $1 | \
+      -e "s|^\($s\)\($w\)$s:$s\[$s\(.*\)$s\]|\1\2:\n\1  - \3|;p" "$1" | \
   sed -ne "s|,$s}$s\$|}|" \
-      -e ":1;s|^\($s\)-$s{$s\(.*\)$s,$s\($w\)$s:$s\(.*\)$s}|\1- {\2}\n\1  \3: \4|;t1" \
-      -e    "s|^\($s\)-$s{$s\(.*\)$s}|\1-\n\1  \2|;p" | \
+      -e "s|^\($s\)-$s{$s\(.*\)$s}|\1-\n\1  \2|;p" | \
   sed -ne "s|^\($s\):|\1|" \
-      -e "s|^\($s\)-$s[\"']\(.*\)[\"']$s\$|\1$fs$fs\2|p" \
+      -e "s|^\($s\)-$s{[\"']\(.*\)[\"']}$s\$|\1$fs$fs\2|p" \
       -e "s|^\($s\)-$s\(.*\)$s\$|\1$fs$fs\2|p" \
-      -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \
+      -e "s|^\($s\)\($w\)$s:$s{[\"']\(.*\)[\"']}$s\$|\1$fs\2$fs\3|p" \
       -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" | \
-  awk -F$fs '{
+  awk -F"$fs" '{
     indent = length($1)/2;
     vname[indent] = $2;
     for (i in vname) {if (i > indent) {delete vname[i]; idx[i]=0}}
     if (length($2) == 0) {vname[indent] = ++idx[indent] };
     if (length($3) > 0) {
       vn=""; for (i=0; i<indent; i++) {vn = (vn)(vname[i])("_")}
-      if ("'$2'_" == vn) {
+      if ("'"$2"'_" == vn) {
          print substr($3 ,1 , match($3,":")-1)
       }
     }
@@ -27,12 +33,12 @@ function parseYaml {
 }
 
 # Check if volume files exist to avoid creating an empty folder
-VOLUME_LIST=`parseYaml docker-compose.dev.yml services_app_volumes`
+VOLUME_LIST=$(parseYaml docker-compose.dev.yml services_app_volumes)
 IGNORE_LIST="./src/app/code ./src/m2-hotfixes ./src/patches ./src/var/log ./src/var/report ./src"
 IS_VALID=true
 # Loop through all files missing from the docker-compose.dev.yml file
 for file in $VOLUME_LIST; do
-  if [ ! -e $file ] && [[ ! " $IGNORE_LIST " =~ " $file " ]]; then
+  if [ ! -e "$file" ] && [[ ! " $IGNORE_LIST " =~ $file ]]; then
     echo "$file: No such file or directory"
     IS_VALID=false
   fi
@@ -48,3 +54,6 @@ docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --remove-or
 #bin/root blackfire-agent --register --server-id={YOUR_SERVER_ID} --server-token={YOUR_SERVER_TOKEN}
 ## Then uncomment the below line (and leave uncommented) to start the agent automatically with bin/start:
 #bin/root /etc/init.d/blackfire-agent start
+
+bin/devtools-cli-check
+bin/cliq /var/www/.composer-global/vendor/bin/cache-clean.js --quiet --watch &

compose/bin/update

@@ -1,5 +1,6 @@
 #!/bin/bash
-mkdir -p tmpupdate && cd $_
+set -o errexit
+mkdir -p tmpupdate && cd "$_"
 curl -s https://raw.githubusercontent.com/markshust/docker-magento/master/lib/template | bash
 rm -rf .git
 rsync -av ./ ../

compose/bin/xdebug

@@ -1,14 +1,55 @@
 #!/bin/bash
-if [ "$1" == "disable" ]; then
-  bin/cli sed -i -e 's/^zend_extension/\;zend_extension/g' /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini
-  sleep 1
-  bin/restart phpfpm
-  echo "Xdebug has been disabled."
-elif [ "$1" == "enable" ]; then
-  bin/cli sed -i -e 's/^\;zend_extension/zend_extension/g' /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini
-  sleep 1
-  bin/restart phpfpm
-  echo "Xdebug has been enabled."
+
+S=$(bin/cli cat /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini | grep -iGc '^;');
+
+xdebug_status() {
+    if [[ $S == 1 ]]; then
+        echo "Xdebug is disabled."
+    else
+        echo "Xdebug is enabled."
+    fi
+}
+
+xdebug_toggle() {
+    if [[ $S == 1 ]]; then
+        xdebug_enable
+    else
+        xdebug_disable
+    fi
+}
+
+xdebug_enable() {
+    if [[ $S == 1 ]]; then
+        bin/cli sed -i -e 's/^\;zend_extension/zend_extension/g' /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini
+        sleep 1
+        bin/restart phpfpm
+        echo "Xdebug has been enabled."
+    else
+        echo "Xdebug is already enabled."
+    fi
+}
+
+xdebug_disable() {
+    if [[ $S == 0 ]]; then
+        bin/cli sed -i -e 's/^\zend_extension/;zend_extension/g' /usr/local/etc/php/conf.d/docker-php-ext-xdebug.ini
+        sleep 1
+        bin/restart phpfpm
+        echo "Xdebug has been disabled."
+    else
+        echo "Xdebug is already disabled."
+    fi
+}
+
+firstArgLetter="$(echo "$1" | head -c 1)"
+
+if [[ $firstArgLetter == "d" ]]; then
+    xdebug_disable
+elif [[ $firstArgLetter == "e" ]]; then
+    xdebug_enable
+elif [[ $firstArgLetter == "t" ]]; then
+    xdebug_toggle
+elif [[ $firstArgLetter == "s" ]]; then
+    xdebug_status
 else
-  echo "Please specify either 'enable' or 'disable' as an argument"
+    printf "Please specify either 'disable', 'enable', 'status' or 'toggle' as an argument.\nEx: bin/xdebug status\n"
 fi

compose/docker-compose.dev.yml

@@ -1,7 +1,7 @@
 # Mark Shust's Docker Configuration for Magento
 # (https://github.com/markshust/docker-magento)
 #
-# Version 34.2.0
+# Version 36.0.1
 
 version: "3"
 

compose/docker-compose.yml

@@ -1,7 +1,7 @@
 # Mark Shust's Docker Configuration for Magento
 # (https://github.com/markshust/docker-magento)
 #
-# Version 34.2.0
+# Version 36.0.1
 
 version: "3"
 
@@ -21,7 +21,7 @@ services:
       - ssldata:/etc/nginx/certs
 
   phpfpm:
-    image: markoshust/magento-php:7.4-fpm-2
+    image: markoshust/magento-php:7.4-fpm-3
     links:
       - db
     volumes: *appvolumes
@@ -39,7 +39,7 @@ services:
     image: redis:5.0-alpine
 
   elasticsearch:
-    image: markoshust/magento-elasticsearch:7.6.2-2
+    image: markoshust/magento-elasticsearch:7.7.1-0
     ports:
       - "9200:9200"
       - "9300:9300"
@@ -56,7 +56,7 @@ services:
 
   # Disabling cron by default as it uses higher CPU, enable if needed
   #cron:
-  #  image: markoshust/magento-php:7.4-fpm-2
+  #  image: markoshust/magento-php:7.4-fpm-3
   #  user: root
   #  command: /usr/local/bin/cronstart
   #  tty: true

images/elasticsearch/7.7/Dockerfile

@@ -0,0 +1,4 @@
+FROM elasticsearch:7.7.1
+
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install analysis-icu
+RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install analysis-phonetic

images/php/7.3/Dockerfile

@@ -76,7 +76,7 @@ RUN groupadd -g 1000 app \
  && useradd -g 1000 -u 1000 -d /var/www -s /bin/bash app
 
 RUN apt-get install -y gnupg \
-  && curl -sL https://deb.nodesource.com/setup_10.x | bash - \
+  && curl -sL https://deb.nodesource.com/setup_14.x | bash - \
   && apt-get install -y nodejs \
   && mkdir /var/www/.config /var/www/.npm \
   && chown app:app /var/www/.config /var/www/.npm \
@@ -89,6 +89,10 @@ RUN curl -sSLO https://github.com/mailhog/mhsendmail/releases/download/v0.2.0/mh
 RUN curl -sS https://getcomposer.org/installer | \
   php -- --1 --install-dir=/usr/local/bin --filename=composer
 
+RUN mkdir /var/www/.composer-global \
+  && composer init --working-dir=/var/www/.composer-global --no-interaction \
+  && composer require --working-dir=/var/www/.composer-global markshust/magento2-metapackage-devtools-cli:^1.0
+
 RUN curl -s https://packages.blackfire.io/gpg.key | apt-key add - \
   && echo "deb http://packages.blackfire.io/debian any main" | tee /etc/apt/sources.list.d/blackfire.list \
   && apt-get update \

images/php/7.4/Dockerfile

@@ -76,7 +76,7 @@ RUN groupadd -g 1000 app \
  && useradd -g 1000 -u 1000 -d /var/www -s /bin/bash app
 
 RUN apt-get install -y gnupg \
-  && curl -sL https://deb.nodesource.com/setup_10.x | bash - \
+  && curl -sL https://deb.nodesource.com/setup_14.x | bash - \
   && apt-get install -y nodejs \
   && mkdir /var/www/.config /var/www/.npm \
   && chown app:app /var/www/.config /var/www/.npm \
@@ -89,6 +89,10 @@ RUN curl -sSLO https://github.com/mailhog/mhsendmail/releases/download/v0.2.0/mh
 RUN curl -sS https://getcomposer.org/installer | \
   php -- --1 --install-dir=/usr/local/bin --filename=composer
 
+RUN mkdir /var/www/.composer-global \
+  && composer init --working-dir=/var/www/.composer-global --no-interaction \
+  && composer require --working-dir=/var/www/.composer-global markshust/magento2-metapackage-devtools-cli:^1.0
+
 RUN curl -s https://packages.blackfire.io/gpg.key | apt-key add - \
   && echo "deb http://packages.blackfire.io/debian any main" | tee /etc/apt/sources.list.d/blackfire.list \
   && apt-get update \

images/php/8.0/Dockerfile

@@ -55,7 +55,7 @@ RUN groupadd -g 1000 app \
  && useradd -g 1000 -u 1000 -d /var/www -s /bin/bash app
 
 RUN apt-get install -y gnupg \
-  && curl -sL https://deb.nodesource.com/setup_10.x | bash - \
+  && curl -sL https://deb.nodesource.com/setup_14.x | bash - \
   && apt-get install -y nodejs \
   && mkdir /var/www/.config /var/www/.npm \
   && chown app:app /var/www/.config /var/www/.npm \
@@ -68,6 +68,10 @@ RUN curl -sSLO https://github.com/mailhog/mhsendmail/releases/download/v0.2.0/mh
 RUN curl -sS https://getcomposer.org/installer | \
   php -- --1 --install-dir=/usr/local/bin --filename=composer
 
+RUN mkdir /var/www/.composer-global \
+  && composer init --working-dir=/var/www/.composer-global --no-interaction \
+  && composer require --working-dir=/var/www/.composer-global markshust/magento2-metapackage-devtools-cli:^1.0
+
 RUN curl -s https://packages.blackfire.io/gpg.key | apt-key add - \
   && echo "deb http://packages.blackfire.io/debian any main" | tee /etc/apt/sources.list.d/blackfire.list \
   && apt-get update \

lib/onelinesetup

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -o errexit
 DOMAIN=${1:-magento2.test}
 VERSION=${2:-2.4.0}
 EDITION=${3:-community}