diff --git a/.gitlab/.ci-linters.yml b/.gitlab/.ci-linters.yml index 6bf140fbe61831..5d36e3d298fa74 100644 --- a/.gitlab/.ci-linters.yml +++ b/.gitlab/.ci-linters.yml @@ -30,6 +30,7 @@ needs-rules: - lint_linux-arm64 - lint_linux-x64 - lint_macos_gitlab_amd64 + - lint_macos_gitlab_arm64 - new-e2e-eks-cleanup-on-failure - protobuf_test - publish_winget_7_x64 @@ -39,6 +40,7 @@ needs-rules: - tests_ebpf_arm64 - tests_ebpf_x64 - tests_macos_gitlab_amd64 + - tests_macos_gitlab_arm64 - tests_windows_secagent_x64 - tests_windows_sysprobe_x64 - trigger_auto_staging_release diff --git a/.gitlab/deploy_packages/deploy_common.yml b/.gitlab/deploy_packages/deploy_common.yml index e51fcc123d4e60..4eec585e24fbb6 100644 --- a/.gitlab/deploy_packages/deploy_common.yml +++ b/.gitlab/deploy_packages/deploy_common.yml @@ -24,6 +24,17 @@ variables: MAJOR_VERSION: 7 +.deploy_packages_dmg: + stage: deploy_packages + rules: + !reference [.on_deploy] + image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX:$CI_IMAGE_GITLAB_AGENT_DEPLOY + tags: ["arch:amd64"] + before_script: + - ls $OMNIBUS_PACKAGE_DIR + script: + - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-7*.dmg" $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/dmg/$PACKAGE_ARCH/ + .deploy_packages_rpm: resource_group: rpm_bucket image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX:$CI_IMAGE_GITLAB_AGENT_DEPLOY diff --git a/.gitlab/deploy_packages/nix.yml b/.gitlab/deploy_packages/nix.yml index 1fd7d41af5dc9d..f726904f728eaf 100644 --- a/.gitlab/deploy_packages/nix.yml +++ b/.gitlab/deploy_packages/nix.yml @@ -147,16 +147,16 @@ deploy_packages_dogstatsd_suse_rpm-x64-7: PACKAGE_ARCH: x86_64 deploy_packages_dmg-x64-a7: - rules: - !reference [.on_deploy] - stage: deploy_packages - image: registry.ddbuild.io/ci/datadog-agent-buildimages/gitlab_agent_deploy$CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX:$CI_IMAGE_GITLAB_AGENT_DEPLOY - tags: ["arch:amd64"] + extends: .deploy_packages_dmg needs: ["agent_dmg-x64-a7"] - before_script: - - ls $OMNIBUS_PACKAGE_DIR - script: - - $S3_CP_CMD --recursive --exclude "*" --include "datadog-agent-7*.dmg" $OMNIBUS_PACKAGE_DIR $S3_RELEASE_ARTIFACTS_URI/dmg/x86_64/ + variables: + PACKAGE_ARCH: x86_64 + +deploy_packages_dmg-arm64-a7: + extends: .deploy_packages_dmg + needs: ["agent_dmg-arm64-a7"] + variables: + PACKAGE_ARCH: arm64 # deploy dogstatsd x64, non-static binary to staging bucket deploy_staging_dsd: diff --git a/.gitlab/functional_test/static_quality_gate.yml b/.gitlab/functional_test/static_quality_gate.yml index b2d80d8b7a0745..7d491ccf655e6e 100644 --- a/.gitlab/functional_test/static_quality_gate.yml +++ b/.gitlab/functional_test/static_quality_gate.yml @@ -49,9 +49,6 @@ static_quality_gates: - iot_agent_rpm-armhf - iot_agent_suse-x64 - windows_msi_and_bosh_zip_x64-a7 - # - job: agent_dmg-x64-a7 - # optional: True - # script: - !reference [.login_to_docker_readonly_crane] # `datadog-ci` relies on `DATADOG_API_KEY` so we get that here. diff --git a/.gitlab/lint/macos.yml b/.gitlab/lint/macos.yml index 4f5100159e40f4..4135356d80934c 100644 --- a/.gitlab/lint/macos.yml +++ b/.gitlab/lint/macos.yml @@ -6,6 +6,9 @@ include: stage: lint extends: .macos_gitlab needs: ["go_deps", "go_tools_deps"] + rules: + - !reference [.except_mergequeue] + - when: on_success script: - !reference [.retrieve_linux_go_deps] - !reference [.retrieve_linux_go_tools_deps] @@ -14,13 +17,7 @@ include: lint_macos_gitlab_amd64: extends: .lint_macos_gitlab tags: ["macos:ventura-amd64", "specific:true"] - rules: - - !reference [.except_mergequeue] - - when: on_success lint_macos_gitlab_arm64: extends: .lint_macos_gitlab - rules: - - !reference [.on_main] - - !reference [.manual] tags: ["macos:ventura-arm64", "specific:true"] diff --git a/.gitlab/package_build/dmg.yml b/.gitlab/package_build/dmg.yml index d28d3233b466ea..63b742ea08d712 100644 --- a/.gitlab/package_build/dmg.yml +++ b/.gitlab/package_build/dmg.yml @@ -23,9 +23,8 @@ export GOMODCACHE=~/gomodcache mkdir -p $GOMODCACHE -agent_dmg-x64-a7: +.agent_dmg: stage: package_build - tags: ["macos:ventura-amd64", "specific:true"] needs: ["go_mod_tidy_check"] rules: - !reference [.on_macos_gui_change] @@ -71,3 +70,11 @@ agent_dmg-x64-a7: - !reference [.macos_setup_cache] - bash .gitlab/package_build/build_agent_dmg.sh - !reference [.upload_sbom_artifacts] + +agent_dmg-x64-a7: + extends: .agent_dmg + tags: ["macos:ventura-amd64", "specific:true"] + +agent_dmg-arm64-a7: + extends: .agent_dmg + tags: ["macos:ventura-arm64", "specific:true"] diff --git a/.gitlab/source_test/macos.yml b/.gitlab/source_test/macos.yml index 768132296de6dd..1f8655597bc8b8 100644 --- a/.gitlab/source_test/macos.yml +++ b/.gitlab/source_test/macos.yml @@ -21,6 +21,11 @@ include: - if [[ "$FAST_TESTS" == "true" ]]; then FAST_TESTS_FLAG="--only-impacted-packages"; fi - dda inv -- -e test --rerun-fails=2 --race --profile --cpus 12 --result-json $TEST_OUTPUT_FILE --junit-tar "junit-${CI_JOB_NAME}.tgz" $FAST_TESTS_FLAG --test-washer --coverage - dda inv -- -e invoke-unit-tests + after_script: + - !reference [.vault_login] + - !reference [.select_python_env_commands] + - !reference [.upload_junit_source] + - !reference [.upload_coverage] artifacts: expire_in: 2 weeks when: always @@ -35,19 +40,7 @@ include: tests_macos_gitlab_amd64: extends: .tests_macos_gitlab tags: ["macos:ventura-amd64", "specific:true"] - after_script: - - !reference [.vault_login] - - !reference [.select_python_env_commands] - - !reference [.upload_junit_source] - - !reference [.upload_coverage] tests_macos_gitlab_arm64: extends: .tests_macos_gitlab - rules: - !reference [.manual] tags: ["macos:ventura-arm64", "specific:true"] - after_script: - - !reference [.vault_login] - - !reference [.select_python_env_commands] - - !reference [.upload_junit_source] - - !reference [.upload_coverage] diff --git a/omnibus/config/software/datadog-agent-integrations-py3.rb b/omnibus/config/software/datadog-agent-integrations-py3.rb index 1c25ca4cf22738..583ffd34b3322d 100644 --- a/omnibus/config/software/datadog-agent-integrations-py3.rb +++ b/omnibus/config/software/datadog-agent-integrations-py3.rb @@ -86,7 +86,7 @@ when linux_target? arm_target? ? "linux-aarch64" : "linux-x86_64" when osx_target? - "macos-x86_64" + arm_target? ? "macos-aarch64" : "macos-x86_64" when windows_target? "windows-x86_64" end + "_#{python_version}.txt" diff --git a/omnibus/config/software/datadog-agent.rb b/omnibus/config/software/datadog-agent.rb index 7bf2c89eaced4d..b1e2a8d23f6fcb 100644 --- a/omnibus/config/software/datadog-agent.rb +++ b/omnibus/config/software/datadog-agent.rb @@ -256,7 +256,8 @@ systray_build_dir = "#{project_dir}/comp/core/gui/guiimpl/systray" # Target OSX 10.10 (it brings significant changes to Cocoa and Foundation APIs, and older versions of OSX are EOL'ed) # Add @executable_path/../Frameworks to rpath to find the swift libs in the Frameworks folder. - command 'swiftc -O -swift-version "5" -target "x86_64-apple-macosx10.10" -Xlinker \'-rpath\' -Xlinker \'@executable_path/../Frameworks\' Sources/*.swift -o gui', cwd: systray_build_dir + target = arm_target? ? 'arm64-apple-macosx11.0' : 'x86_64-apple-macosx10.10' + command "swiftc -O -swift-version \"5\" -target \"#{target}\" -Xlinker '-rpath' -Xlinker '@executable_path/../Frameworks' Sources/*.swift -o gui", cwd: systray_build_dir copy "#{systray_build_dir}/gui", "#{app_temp_dir}/MacOS/" copy "#{systray_build_dir}/agent.png", "#{app_temp_dir}/MacOS/" end diff --git a/omnibus/config/software/openssl3.rb b/omnibus/config/software/openssl3.rb index cdadb6f8f318ea..53fecebb806f2e 100644 --- a/omnibus/config/software/openssl3.rb +++ b/omnibus/config/software/openssl3.rb @@ -60,7 +60,7 @@ configure_args = [] if mac_os_x? configure_cmd = "./Configure" - configure_args << "darwin64-x86_64-cc" + configure_args << "darwin64-#{arm_target? ? "arm64" : "x86_64"}-cc" elsif windows? configure_cmd = "perl.exe ./Configure" configure_args << (windows_arch_i386? ? "mingw" : "mingw64") diff --git a/omnibus/config/software/python3.rb b/omnibus/config/software/python3.rb index 4d869290527ee1..010b8fbd23c519 100644 --- a/omnibus/config/software/python3.rb +++ b/omnibus/config/software/python3.rb @@ -32,7 +32,7 @@ if mac_os_x? python_configure_options.push("--enable-ipv6", - "--with-universal-archs=intel", + "--with-universal-archs=#{arm_target? ? "universal2" : "intel"}", "--enable-shared") elsif linux_target? python_configure_options.push("--enable-shared", diff --git a/pkg/process/checks/host_info_test.go b/pkg/process/checks/host_info_test.go index eb733d3431f5a2..90a827159e485d 100644 --- a/pkg/process/checks/host_info_test.go +++ b/pkg/process/checks/host_info_test.go @@ -100,7 +100,7 @@ func TestGetHostnameFromCmd(t *testing.T) { } func TestResolveHostname(t *testing.T) { - if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" { + if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" { t.Skip("TestResolveHostname is known to fail on the macOS Gitlab runners because of the already running Agent") } osHostname, err := os.Hostname() diff --git a/pkg/serverless/daemon/daemon_test.go b/pkg/serverless/daemon/daemon_test.go index dab72e9d4c6cf4..2dfa2447cf0173 100644 --- a/pkg/serverless/daemon/daemon_test.go +++ b/pkg/serverless/daemon/daemon_test.go @@ -185,7 +185,7 @@ func TestSetTraceTagOk(t *testing.T) { } func TestOutOfOrderInvocations(t *testing.T) { - if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" { + if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" { t.Skip("TestOutOfOrderInvocations is known to fail on the macOS Gitlab runners because of the already running Agent") } port := testutil.FreeTCPPort(t) diff --git a/pkg/serverless/metrics/metric_test.go b/pkg/serverless/metrics/metric_test.go index 460d0930da2656..04ed35d6408f60 100644 --- a/pkg/serverless/metrics/metric_test.go +++ b/pkg/serverless/metrics/metric_test.go @@ -39,7 +39,7 @@ func TestMain(m *testing.M) { } func TestStartDoesNotBlock(t *testing.T) { - if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" { + if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" { t.Skip("TestStartDoesNotBlock is known to fail on the macOS Gitlab runners because of the already running Agent") } mockConfig := configmock.New(t) @@ -118,7 +118,7 @@ func TestStartWithProxy(t *testing.T) { } func TestRaceFlushVersusAddSample(t *testing.T) { - if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" { + if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" { t.Skip("TestRaceFlushVersusAddSample is known to fail on the macOS Gitlab runners because of the already running Agent") } metricAgent := &ServerlessMetricAgent{ diff --git a/pkg/serverless/proxy/proxy_test.go b/pkg/serverless/proxy/proxy_test.go index 8bd014ea7de172..d461c4000d9a19 100644 --- a/pkg/serverless/proxy/proxy_test.go +++ b/pkg/serverless/proxy/proxy_test.go @@ -69,7 +69,7 @@ func (tp *testProcessorResponseError) GetExecutionInfo() *invocationlifecycle.Ex } func TestProxyResponseValid(t *testing.T) { - if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" && runtime.GOARCH == "amd64" { + if os.Getenv("CI") == "true" && runtime.GOOS == "darwin" { t.Skip("TestProxyResponseValid is known to fail on the macOS Gitlab runners because of the already running Agent") } // fake the runtime API running on 5001 diff --git a/releasenotes/notes/add-macos-build-for-aarch64-767531f392e77140.yaml b/releasenotes/notes/add-macos-build-for-aarch64-767531f392e77140.yaml new file mode 100644 index 00000000000000..872f83c0803924 --- /dev/null +++ b/releasenotes/notes/add-macos-build-for-aarch64-767531f392e77140.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Add macOS build for AArch64/ARM64. diff --git a/tasks/omnibus.py b/tasks/omnibus.py index 2842053bbc8a10..8815614d7215bf 100644 --- a/tasks/omnibus.py +++ b/tasks/omnibus.py @@ -109,8 +109,7 @@ def get_omnibus_env( env[key] = value if sys.platform == 'darwin': - # Target MacOS 10.12 - env['MACOSX_DEPLOYMENT_TARGET'] = '10.12' + env['MACOSX_DEPLOYMENT_TARGET'] = '11.0' if os.uname().machine == "arm64" else '10.12' if skip_sign: env['SKIP_SIGN_MAC'] = 'true'