M1 mac compatibility (#5511)

* Apply x86 protoc for m1 mac

* Enable the Dockerfile suffix for m1 Mac

* Enable the Dockerfile suffix for ow-utils

* Choose zookeeper image according to the os architecture

* Use the bitnami/etcd image that supports arm

* Replace docker info API to cli command

ansible/group_vars/all

@@ -186,7 +186,9 @@ kafka_connect_string: "{% set ret = [] %}\
 kafka_protocol_for_setup: "{{ kafka_protocol | default('PLAINTEXT') }}"
 
 zookeeper:
-  version: 3.4
+  image:
+    amd64: zookeeper:3.4
+    arm64: arm64v8/zookeeper:3.4
   port: 2181
 
 zookeeper_connect_string: "{% set ret = [] %}\
@@ -464,7 +466,7 @@ zeroDowntimeDeployment:
   enabled: "{{ zerodowntime_deployment_switch | default(false) }}"
 
 etcd:
-  version: "{{ etcd_version | default('v3.4.33') }}"
+  version: "{{ etcd_version | default('3.5') }}"
   client:
     port: 2379
   server:

ansible/roles/etcd/tasks/deploy.yml

@@ -38,24 +38,13 @@
 - name: (re)start etcd
   docker_container:
     name: etcd{{ groups['etcd'].index(inventory_hostname) }}
-    image: quay.io/coreos/etcd:{{ etcd.version }}
+    image: bitnami/etcd:{{ etcd.version }}
     state: started
     recreate: true
     restart_policy: "{{ docker.restart.policy }}"
     volumes: "{{volume_dir | default([])}}"
-    command: "/usr/local/bin/etcd \
-               --data-dir=/etcd-data --name '{{ name }}' \
-               --initial-advertise-peer-urls http://{{ ansible_host }}:{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }} \
-               --advertise-client-urls http://{{ ansible_host }}:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }} \
-               --listen-peer-urls http://0.0.0.0:{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }} \
-               --listen-client-urls http://0.0.0.0:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }} \
-               --initial-cluster {{ cluster }} \
-               --initial-cluster-state new --initial-cluster-token {{ etcd.cluster.token }} \
-               --quota-backend-bytes {{ etcd.quota_backend_bytes }} \
-               --snapshot-count {{ etcd.snapshot_count }} \
-               --auto-compaction-retention {{ etcd.auto_compaction_retention }} \
-               --auto-compaction-mode {{ etcd.auto_compaction_mode }} \
-               --log-level {{ etcd.loglevel }}"
+    env:
+     "ALLOW_NONE_AUTHENTICATION": "yes"
     ports:
       - "{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }}:{{ etcd.client.port + groups['etcd'].index(inventory_hostname) }}"
       - "{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }}:{{ etcd.server.port + groups['etcd'].index(inventory_hostname) }}"

ansible/roles/invoker/tasks/deploy.yml

@@ -80,7 +80,7 @@
   when: environmentInformation.type == "docker-machine"
 
 - name: "determine docker root dir"
-  shell: echo -e "GET http:/v1.24/info HTTP/1.0\r\n" | nc -U /var/run/docker.sock | grep "{"
+  shell: docker info -f json
   args:
     executable: /bin/bash
   register: dockerInfo_output

ansible/roles/zookeeper/tasks/deploy.yml

@@ -17,10 +17,20 @@
 ---
 # This role will install Kafka with Zookeeper in group 'kafka' in the environment inventory
 
+- name: set zookeeper image
+  set_fact:
+    zookeeper_image: "{{ zookeeper.image.arm64 }}"
+  when: ansible_facts['os_family'] == "Darwin" and ansible_facts['architecture'] == "arm64"
+
+- name: set zookeeper image
+  set_fact:
+    zookeeper_image: "{{ zookeeper.image.amd64 }}"
+  when: ansible_facts['os_family'] != "Darwin" or ansible_facts['architecture'] != "arm64"
+
 - name: (re)start zookeeper
   docker_container:
     name: zookeeper{{ groups['zookeepers'].index(inventory_hostname) }}
-    image: zookeeper:{{ zookeeper.version }}
+    image: "{{ zookeeper_image }}"
     state: started
     recreate: true
     restart_policy: "{{ docker.restart.policy }}"

common/scala/Dockerfile.arm

@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# if you change version of openjsk, also update tools/github/setup.sh to download the corresponding jdk
+FROM arm64v8/eclipse-temurin:21.0.4_7-jdk-alpine
+
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+# Switch to the HTTPS endpoint for the apk repositories as per https://github.com/gliderlabs/docker-alpine/issues/184
+RUN sed -i 's/http\:\/\/dl-cdn.alpinelinux.org/https\:\/\/alpine.global.ssl.fastly.net/g' /etc/apk/repositories
+RUN apk add --update sed curl bash && apk update && apk upgrade
+
+RUN mkdir /logs
+
+COPY transformEnvironment.sh /
+RUN chmod +x transformEnvironment.sh
+
+COPY copyJMXFiles.sh /
+RUN chmod +x copyJMXFiles.sh

common/scala/build.gradle

@@ -24,6 +24,9 @@ plugins {
 }
 
 ext.dockerImageName = 'scala'
+if(System.getProperty("os.arch").toLowerCase(Locale.ENGLISH).startsWith("aarch")) {
+    ext.dockerDockerfileSuffix = ".arm"
+}
 apply from: '../../gradle/docker.gradle'
 
 project.archivesBaseName = "openwhisk-common"

core/scheduler/build.gradle

@@ -52,6 +52,16 @@ buildscript {
     }
 }
 
+protobuf {
+    protoc {
+        if (osdetector.os == "osx") {
+            artifact = 'com.google.protobuf:protoc:3.4.0:osx-x86_64'
+        } else {
+            artifact = 'com.google.protobuf:protoc:3.4.0'
+        }
+    }
+}
+
 // Define a separate configuration for managing the dependency on Jetty ALPN agent.
 configurations {
     alpnagent

gradle/docker.gradle

@@ -46,7 +46,7 @@ ext {
     dockerRetries = project.hasProperty('dockerRetries') ? dockerRetries.toInteger() : 3
     dockerBinary = project.hasProperty('dockerBinary') ? [dockerBinary] : ['docker']
     dockerBuildArg = ['build']
-    dockerDockerfileSuffix = ""
+    dockerDockerfileSuffix = project.hasProperty('dockerDockerfileSuffix') ? dockerDockerfileSuffix : ""
 }
 ext.dockerTaggedImageName = dockerRegistry + dockerImagePrefix + '/' + dockerImageName + ':' + dockerImageTag
 
@@ -60,10 +60,6 @@ if(project.hasProperty('dockerBuildArgs')) {
     }
 }
 
-if(project.hasProperty('dockerfileSuffix')) {
-    dockerDockerfileSuffix = project.dockerfileSuffix
-}
-
 task distDocker {
     doLast {
         def start = new Date()

tools/ow-utils/Dockerfile.arm

@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM arm64v8/eclipse-temurin:8u422-b05-jdk-noble
+
+ENV DOCKER_VERSION 1.12.0
+ENV KUBECTL_VERSION v1.16.3
+ENV WHISK_CLI_VERSION latest
+ENV WHISKDEPLOY_CLI_VERSION latest
+
+RUN apt-get update && apt-get install -y \
+  git \
+  jq \
+  libffi-dev \
+  nodejs \
+  npm \
+  python-is-python3 \
+  python3-pip \
+  python3-venv \
+  wget \
+  zip \
+  locales \
+&& rm -rf /var/lib/apt/lists/*
+
+# update npm
+RUN npm install -g n && n stable && hash -r
+
+RUN locale-gen en_US.UTF-8
+ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
+
+WORKDIR /root
+
+RUN python -m venv .venv
+ENV PATH="/root/.venv/bin:$PATH"
+
+# Python packages
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install cryptography && \
+    pip3 install ansible==2.5.2 && \
+    pip3 install jinja2==2.9.6 && \
+    pip3 install docker
+
+# Install docker client
+RUN wget --no-verbose https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz && \
+  tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker && \
+  rm -f docker-${DOCKER_VERSION}.tgz && \
+  chmod +x /usr/bin/docker
+
+# Install kubectl in /usr/local/bin
+RUN curl -Lo ./kubectl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/kubectl
+
+# Install `wsk` cli in /usr/local/bin
+RUN wget -q https://github.com/apache/openwhisk-cli/releases/download/$WHISK_CLI_VERSION/OpenWhisk_CLI-$WHISK_CLI_VERSION-linux-amd64.tgz && \
+  tar xzf OpenWhisk_CLI-$WHISK_CLI_VERSION-linux-amd64.tgz -C /usr/local/bin wsk && \
+  rm OpenWhisk_CLI-$WHISK_CLI_VERSION-linux-amd64.tgz
+
+# Install wskadmin in /bin
+COPY wskutil.py /bin
+COPY wskprop.py /bin
+COPY wskadmin /bin
+
+# Setup tools/data for certificate generation (used by openwhisk-deploy-kube)
+RUN mkdir /cert-gen
+COPY openwhisk-server-key.pem /cert-gen
+COPY genssl.sh /usr/local/bin/

tools/ow-utils/build.gradle

@@ -16,6 +16,9 @@
  */
 
 ext.dockerImageName = 'ow-utils'
+if(System.getProperty("os.arch").toLowerCase(Locale.ENGLISH).startsWith("aarch64")) {
+    ext.dockerDockerfileSuffix = ".arm"
+}
 apply from: '../../gradle/docker.gradle'
 
 distDocker.dependsOn 'copyWskadmin', 'copyGenssl'