[ABLD-28] Add macOS build for AArch64/ARM64

.gitlab/.ci-linters.yml

@@ -30,6 +30,7 @@ needs-rules:
     - lint_linux-arm64
     - lint_linux-x64
     - lint_macos_gitlab_amd64
+    - lint_macos_gitlab_arm64
     - new-e2e-eks-cleanup-on-failure
     - protobuf_test
     - publish_winget_7_x64
@@ -39,6 +40,7 @@ needs-rules:
     - tests_ebpf_arm64
     - tests_ebpf_x64
     - tests_macos_gitlab_amd64
+    - tests_macos_gitlab_arm64
     - tests_windows_secagent_x64
     - tests_windows_sysprobe_x64
     - trigger_auto_staging_release

.gitlab/deploy_packages/deploy_common.yml

@@ -24,6 +24,17 @@
   variables:
     MAJOR_VERSION: 7
 
+.deploy_packages_dmg:
+  stage: deploy_packages
+  rules:
+    !reference [.on_deploy]
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX:$CI_IMAGE_GITLAB_AGENT_DEPLOY
+  tags: ["arch:amd64"]
+  before_script:
+    - ls $OMNIBUS_PACKAGE_DIR
+  script:
+    - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-7*.dmg" $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/dmg/$PACKAGE_ARCH/
+
 .deploy_packages_rpm:
   resource_group: rpm_bucket
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX:$CI_IMAGE_GITLAB_AGENT_DEPLOY

.gitlab/deploy_packages/nix.yml

@@ -147,16 +147,16 @@ deploy_packages_dogstatsd_suse_rpm-x64-7:
     PACKAGE_ARCH: x86_64
 
 deploy_packages_dmg-x64-a7:
-  rules:
-    !reference [.on_deploy]
-  stage: deploy_packages
-  image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX:$CI_IMAGE_GITLAB_AGENT_DEPLOY
-  tags: ["arch:amd64"]
+  extends: .deploy_packages_dmg
   needs: ["agent_dmg-x64-a7"]
-  before_script:
-    - ls $OMNIBUS_PACKAGE_DIR
-  script:
-    - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-7*.dmg" $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/dmg/x86_64/
+  variables:
+    PACKAGE_ARCH: x86_64
+
+deploy_packages_dmg-arm64-a7:
+  extends: .deploy_packages_dmg
+  needs: ["agent_dmg-arm64-a7"]
+  variables:
+    PACKAGE_ARCH: arm64
 
 # deploy dogstatsd x64, non-static binary to staging bucket
 deploy_staging_dsd:

.gitlab/functional_test/static_quality_gate.yml

@@ -49,9 +49,6 @@ static_quality_gates:
     - iot_agent_rpm-armhf
     - iot_agent_suse-x64
     - windows_msi_and_bosh_zip_x64-a7
- #  - job: agent_dmg-x64-a7
- #    optional: True
- #
   script:
     - !reference [.login_to_docker_readonly_crane]
     # `datadog-ci` relies on `DATADOG_API_KEY` so we get that here.

.gitlab/lint/macos.yml

@@ -6,6 +6,9 @@ include:
   stage: lint
   extends: .macos_gitlab
   needs: ["go_deps", "go_tools_deps"]
+  rules:
+    - !reference [.except_mergequeue]
+    - when: on_success
   script:
     - !reference [.retrieve_linux_go_deps]
     - !reference [.retrieve_linux_go_tools_deps]
@@ -14,13 +17,7 @@ include:
 lint_macos_gitlab_amd64:
   extends: .lint_macos_gitlab
   tags: ["macos:ventura-amd64", "specific:true"]
-  rules:
-    - !reference [.except_mergequeue]
-    - when: on_success
 
 lint_macos_gitlab_arm64:
   extends: .lint_macos_gitlab
-  rules:
-    - !reference [.on_main]
-    - !reference [.manual]
   tags: ["macos:ventura-arm64", "specific:true"]

.gitlab/package_build/dmg.yml

@@ -23,9 +23,8 @@
     export GOMODCACHE=~/gomodcache
     mkdir -p $GOMODCACHE
 
-agent_dmg-x64-a7:
+.agent_dmg:
   stage: package_build
-  tags: ["macos:ventura-amd64", "specific:true"]
   needs: ["go_mod_tidy_check"]
   rules:
      - !reference [.on_macos_gui_change]
@@ -71,3 +70,11 @@ agent_dmg-x64-a7:
     - !reference [.macos_setup_cache]
     - bash .gitlab/package_build/build_agent_dmg.sh
     - !reference [.upload_sbom_artifacts]
+
+agent_dmg-x64-a7:
+  extends: .agent_dmg
+  tags: ["macos:ventura-amd64", "specific:true"]
+
+agent_dmg-arm64-a7:
+  extends: .agent_dmg
+  tags: ["macos:ventura-arm64", "specific:true"]

.gitlab/source_test/macos.yml

@@ -21,6 +21,11 @@ include:
     - if [[ "$FAST_TESTS" == "true" ]]; then FAST_TESTS_FLAG="--only-impacted-packages"; fi
     - dda inv -- -e test --rerun-fails=2 --race --profile --cpus 12 --result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" $FAST_TESTS_FLAG --test-washer --coverage
     - dda inv -- -e invoke-unit-tests
+  after_script:
+    - !reference [.vault_login]
+    - !reference [.select_python_env_commands]
+    - !reference [.upload_junit_source]
+    - !reference [.upload_coverage]
   artifacts:
     expire_in: 2 weeks
     when: always
@@ -35,19 +40,7 @@ include:
 tests_macos_gitlab_amd64:
   extends: .tests_macos_gitlab
   tags: ["macos:ventura-amd64", "specific:true"]
-  after_script:
-    - !reference [.vault_login]
-    - !reference [.select_python_env_commands]
-    - !reference [.upload_junit_source]
-    - !reference [.upload_coverage]
 
 tests_macos_gitlab_arm64:
   extends: .tests_macos_gitlab
-  rules:
-    !reference [.manual]
   tags: ["macos:ventura-arm64", "specific:true"]
-  after_script:
-    - !reference [.vault_login]
-    - !reference [.select_python_env_commands]
-    - !reference [.upload_junit_source]
-    - !reference [.upload_coverage]

omnibus/config/software/datadog-agent-integrations-py3.rb

@@ -86,7 +86,7 @@
     when linux_target?
       arm_target? ? "linux-aarch64" : "linux-x86_64"
     when osx_target?
-      "macos-x86_64"
+      arm_target? ? "macos-aarch64" : "macos-x86_64"
     when windows_target?
       "windows-x86_64"
   end + "_#{python_version}.txt"

omnibus/config/software/datadog-agent.rb

@@ -256,7 +256,8 @@
     systray_build_dir = "#{project_dir}/comp/core/gui/guiimpl/systray"
     # Target OSX 10.10 (it brings significant changes to Cocoa and Foundation APIs, and older versions of OSX are EOL'ed)
     # Add @executable_path/../Frameworks to rpath to find the swift libs in the Frameworks folder.
-    command 'swiftc -O -swift-version "5" -target "x86_64-apple-macosx10.10" -Xlinker \'-rpath\' -Xlinker \'@executable_path/../Frameworks\' Sources/*.swift -o gui', cwd: systray_build_dir
+    target = arm_target? ? 'arm64-apple-macosx11.0' : 'x86_64-apple-macosx10.10'
+    command "swiftc -O -swift-version \"5\" -target \"#{target}\" -Xlinker '-rpath' -Xlinker '@executable_path/../Frameworks' Sources/*.swift -o gui", cwd: systray_build_dir
     copy "#{systray_build_dir}/gui", "#{app_temp_dir}/MacOS/"
     copy "#{systray_build_dir}/agent.png", "#{app_temp_dir}/MacOS/"
   end

omnibus/config/software/openssl3.rb

@@ -60,7 +60,7 @@
   configure_args = []
   if mac_os_x?
     configure_cmd = "./Configure"
-    configure_args << "darwin64-x86_64-cc"
+    configure_args << "darwin64-#{arm_target? ? "arm64" : "x86_64"}-cc"
   elsif windows?
     configure_cmd = "perl.exe ./Configure"
     configure_args << (windows_arch_i386? ? "mingw" : "mingw64")

omnibus/config/software/python3.rb

@@ -32,7 +32,7 @@
 
     if mac_os_x?
       python_configure_options.push("--enable-ipv6",
-                            "--with-universal-archs=intel",
+                            "--with-universal-archs=#{arm_target? ? "universal2" : "intel"}",
                             "--enable-shared")
     elsif linux_target?
       python_configure_options.push("--enable-shared",

pkg/process/checks/host_info_test.go

@@ -100,7 +100,7 @@ func TestGetHostnameFromCmd(t *testing.T) {
 }
 
 func TestResolveHostname(t *testing.T) {
-	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" {
 		t.Skip("TestResolveHostname is known to fail on the macOS Gitlab runners because of the already running Agent")
 	}
 	osHostname, err := os.Hostname()

pkg/serverless/daemon/daemon_test.go

@@ -185,7 +185,7 @@ func TestSetTraceTagOk(t *testing.T) {
 }
 
 func TestOutOfOrderInvocations(t *testing.T) {
-	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" {
 		t.Skip("TestOutOfOrderInvocations is known to fail on the macOS Gitlab runners because of the already running Agent")
 	}
 	port := testutil.FreeTCPPort(t)

pkg/serverless/metrics/metric_test.go

@@ -39,7 +39,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestStartDoesNotBlock(t *testing.T) {
-	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" {
 		t.Skip("TestStartDoesNotBlock is known to fail on the macOS Gitlab runners because of the already running Agent")
 	}
 	mockConfig := configmock.New(t)
@@ -118,7 +118,7 @@ func TestStartWithProxy(t *testing.T) {
 }
 
 func TestRaceFlushVersusAddSample(t *testing.T) {
-	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" {
 		t.Skip("TestRaceFlushVersusAddSample is known to fail on the macOS Gitlab runners because of the already running Agent")
 	}
 	metricAgent := &ServerlessMetricAgent{

pkg/serverless/proxy/proxy_test.go

@@ -69,7 +69,7 @@ func (tp *testProcessorResponseError) GetExecutionInfo() *invocationlifecycle.Ex
 }
 
 func TestProxyResponseValid(t *testing.T) {
-	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" {
+	if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" {
 		t.Skip("TestProxyResponseValid is known to fail on the macOS Gitlab runners because of the already running Agent")
 	}
 	// fake the runtime API running on 5001

releasenotes/notes/add-macos-build-for-aarch64-767531f392e77140.yaml

@@ -0,0 +1,11 @@
+# Each section from every release note are combined when the
+# CHANGELOG.rst is rendered. So the text needs to be worded so that
+# it does not depend on any information only available in another
+# section. This may mean repeating some details, but each section
+# must be readable independently of the other.
+#
+# Each section note must be formatted as reStructuredText.
+---
+enhancements:
+  - |
+    Add macOS build for AArch64/ARM64.

tasks/omnibus.py

@@ -109,8 +109,7 @@ def get_omnibus_env(
             env[key] = value
 
     if sys.platform == 'darwin':
-        # Target MacOS 10.12
-        env['MACOSX_DEPLOYMENT_TARGET'] = '10.12'
+        env['MACOSX_DEPLOYMENT_TARGET'] = '11.0' if os.uname().machine == "arm64" else '10.12'
 
     if skip_sign:
         env['SKIP_SIGN_MAC'] = 'true'