From 9c02d7a8cd09bcec7015efbf05e3a0f586afc9c8 Mon Sep 17 00:00:00 2001 From: Jordan Rose Date: Wed, 11 Jan 2023 17:27:42 -0800 Subject: [PATCH] Pods: Change the podspec to download prebuilds Rather than building the Rust parts of libsignal as part of `pod install`, fetch them from build-artifacts.signal.org. This requires adding ENV['LIBSIGNAL_FFI_PREBUILD_CHECKSUM'] = '...' to the consuming Podfile. The referenced archives are downloaded to ~/Library/Caches/org.signal.libsignal, and are unarchived as part of the build. (The archives are outside the build directory so that a clean build does not require a new download.) Building with LibSignalClient as a local pod is still supported; in that case everything will refer to the local target/ directory instead. Use swift/build_ffi.sh to build as usual. --- .github/workflows/build_and_test.yml | 13 ++++- .github/workflows/lints.yml | 4 +- .github/workflows/slow_tests.yml | 25 +++++++++ LibSignalClient.podspec | 76 +++++++++++++++----------- bin/fetch_archive.py | 79 ++++++++++++++++++++++++++++ swift/README.md | 5 +- swift/build_ffi.sh | 1 + 7 files changed, 169 insertions(+), 34 deletions(-) create mode 100755 bin/fetch_archive.py diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7d524f62..204d184c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -277,7 +277,18 @@ jobs: repository: signalapp/SignalCoreKit path: SignalCoreKit - - run: rustup toolchain install $(cat rust-toolchain) --profile minimal --target x86_64-apple-ios,aarch64-apple-ios,aarch64-apple-ios-sim + - run: rustup toolchain install $(cat rust-toolchain) --profile minimal --target x86_64-apple-ios,aarch64-apple-ios-sim + + # Build only the targets that `pod lib lint` will test building. + - name: Build for x86_64-apple-ios + run: swift/build_ffi.sh --release + env: + CARGO_BUILD_TARGET: x86_64-apple-ios + + - name: Build for aarch64-apple-ios-sim + run: swift/build_ffi.sh --release + env: + CARGO_BUILD_TARGET: aarch64-apple-ios-sim - name: Run pod lint # No import validation because it tries to build unsupported platforms (like 32-bit iOS). diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml index b0ff8a4f..378e2ceb 100644 --- a/.github/workflows/lints.yml +++ b/.github/workflows/lints.yml @@ -17,6 +17,8 @@ jobs: steps: - uses: actions/checkout@v3 - - run: pip3 install flake8 + - run: pip3 install flake8 mypy - run: shellcheck **/*.sh - run: python3 -m flake8 . + # Only include typed Python scripts here. + - run: python3 -m mypy bin/fetch_archive.py --python-version 3.6 --strict diff --git a/.github/workflows/slow_tests.yml b/.github/workflows/slow_tests.yml index 14deb8ab..631b62a4 100644 --- a/.github/workflows/slow_tests.yml +++ b/.github/workflows/slow_tests.yml @@ -124,6 +124,31 @@ jobs: - run: rustup toolchain install $(cat rust-toolchain) --profile minimal --target x86_64-apple-ios,aarch64-apple-ios,aarch64-apple-ios-sim --component rust-src + - name: Build for x86_64-apple-ios + run: swift/build_ffi.sh --release + env: + CARGO_BUILD_TARGET: x86_64-apple-ios + + - name: Build for aarch64-apple-ios + run: swift/build_ffi.sh --release + env: + CARGO_BUILD_TARGET: aarch64-apple-ios + + - name: Build for aarch64-apple-ios-sim + run: swift/build_ffi.sh --release + env: + CARGO_BUILD_TARGET: aarch64-apple-ios-sim + + - name: Build for x86_64-apple-ios-macabi + run: swift/build_ffi.sh --release --build-std + env: + CARGO_BUILD_TARGET: x86_64-apple-ios-macabi + + - name: Build for aarch64-apple-ios-macabi + run: swift/build_ffi.sh --release --build-std + env: + CARGO_BUILD_TARGET: aarch64-apple-ios-macabi + - name: Run pod lint # No import validation because it tries to build unsupported platforms (like 32-bit iOS). run: pod lib lint --verbose --platforms=ios --include-podspecs=SignalCoreKit/SignalCoreKit.podspec --skip-import-validation diff --git a/LibSignalClient.podspec b/LibSignalClient.podspec index 3983a1e4..2cda0df9 100644 --- a/LibSignalClient.podspec +++ b/LibSignalClient.podspec @@ -14,17 +14,14 @@ Pod::Spec.new do |s| s.source = { :git => 'https://github.com/signalapp/libsignal.git', :tag => "v#{s.version}" } s.swift_version = '5' - # We use this to set IPHONEOS_DEPLOYMENT_TARGET below. - # The Rust compiler driver expects this to always be in the form 'major.minor'. - min_deployment_target = '12.2' - s.platform = :ios, min_deployment_target + s.platform = :ios, '12.2' s.dependency 'SignalCoreKit' s.source_files = ['swift/Sources/**/*.swift', 'swift/Sources/**/*.m'] s.preserve_paths = [ - 'target/*/release/libsignal_ffi.a', 'swift/Sources/SignalFfi', + 'bin/fetch_archive.py', ] s.pod_target_xcconfig = { @@ -32,10 +29,17 @@ Pod::Spec.new do |s| # Duplicate this here to make sure the search path is passed on to Swift dependencies. 'SWIFT_INCLUDE_PATHS' => '$(HEADER_SEARCH_PATHS)', + 'LIBSIGNAL_FFI_BUILD_PATH' => 'target/$(CARGO_BUILD_TARGET)/release', + # Store libsignal_ffi.a builds in a project-wide directory + # because we keep simulator and device builds next to each other. + 'LIBSIGNAL_FFI_TEMP_DIR' => '$(PROJECT_TEMP_DIR)/libsignal_ffi', + 'LIBSIGNAL_FFI_LIB_TO_LINK' => '$(LIBSIGNAL_FFI_TEMP_DIR)/$(LIBSIGNAL_FFI_BUILD_PATH)/libsignal_ffi.a', + # Make sure we link the static library, not a dynamic one. - # Use an extra level of indirection because CocoaPods messes with OTHER_LDFLAGS too. - 'LIBSIGNAL_FFI_LIB_IF_NEEDED' => '$(PODS_TARGET_SRCROOT)/target/$(CARGO_BUILD_TARGET)/release/libsignal_ffi.a', - 'OTHER_LDFLAGS' => '$(LIBSIGNAL_FFI_LIB_IF_NEEDED)', + 'OTHER_LDFLAGS' => '$(LIBSIGNAL_FFI_LIB_TO_LINK)', + + 'LIBSIGNAL_FFI_PREBUILD_ARCHIVE' => "libsignal-client-ios-build-v#{s.version}.tar.gz", + 'LIBSIGNAL_FFI_PREBUILD_CHECKSUM' => ENV.fetch('LIBSIGNAL_FFI_PREBUILD_CHECKSUM', ''), 'CARGO_BUILD_TARGET[sdk=iphonesimulator*][arch=arm64]' => 'aarch64-apple-ios-sim', 'CARGO_BUILD_TARGET[sdk=iphonesimulator*][arch=*]' => 'x86_64-apple-ios', @@ -55,37 +59,47 @@ Pod::Spec.new do |s| } s.script_phases = [ - { :name => 'Check libsignal-ffi', - :execution_position => :before_compile, - :script => %q( - test -e "${LIBSIGNAL_FFI_LIB_IF_NEEDED}" && exit 0 - if test -e "${PODS_TARGET_SRCROOT}/swift/build_ffi.sh"; then - echo 'error: libsignal_ffi.a not built; run the following to build it:' >&2 - echo "CARGO_BUILD_TARGET=${CARGO_BUILD_TARGET} \"${PODS_TARGET_SRCROOT}/swift/build_ffi.sh\" --release" >&2 - else - echo 'error: libsignal_ffi.a not built; try re-running `pod install`' >&2 + { name: 'Download and cache libsignal-ffi', + execution_position: :before_compile, + script: %q( + set -euo pipefail + if [ -e "${PODS_TARGET_SRCROOT}/swift/build_ffi.sh" ]; then + # Local development + exit 0 + fi + "${PODS_TARGET_SRCROOT}"/bin/fetch_archive.py -u "https://build-artifacts.signal.org/libraries/${LIBSIGNAL_FFI_PREBUILD_ARCHIVE}" -c "${LIBSIGNAL_FFI_PREBUILD_CHECKSUM}" -o "${USER_LIBRARY_DIR}/Caches/org.signal.libsignal" + ), + }, + { name: 'Extract libsignal-ffi prebuild', + execution_position: :before_compile, + input_files: ['$(USER_LIBRARY_DIR)/Caches/org.signal.libsignal/$(LIBSIGNAL_FFI_PREBUILD_ARCHIVE)'], + output_files: ['$(LIBSIGNAL_FFI_LIB_TO_LINK)'], + script: %q( + set -euo pipefail + rm -rf "${LIBSIGNAL_FFI_TEMP_DIR}" + if [ -e "${PODS_TARGET_SRCROOT}/swift/build_ffi.sh" ]; then + # Local development + ln -fhs "${PODS_TARGET_SRCROOT}" "${LIBSIGNAL_FFI_TEMP_DIR}" + elif [ -e "${SCRIPT_INPUT_FILE_0}" ]; then + mkdir -p "${LIBSIGNAL_FFI_TEMP_DIR}" + cd "${LIBSIGNAL_FFI_TEMP_DIR}" + tar --modification-time -x -f "${SCRIPT_INPUT_FILE_0}" + else + echo 'error: could not download libsignal_ffi.a; please provide LIBSIGNAL_FFI_PREBUILD_CHECKSUM' >&2 + exit 1 fi - false ), } ] - s.prepare_command = %Q( - set -euo pipefail - export IPHONEOS_DEPLOYMENT_TARGET=#{min_deployment_target} - CARGO_BUILD_TARGET=aarch64-apple-ios swift/build_ffi.sh --release - CARGO_BUILD_TARGET=x86_64-apple-ios swift/build_ffi.sh --release - CARGO_BUILD_TARGET=aarch64-apple-ios-sim swift/build_ffi.sh --release - if [[ "${SKIP_CATALYST:-0}" != "1" ]]; then - CARGO_BUILD_TARGET=x86_64-apple-ios-macabi swift/build_ffi.sh --release --build-std - CARGO_BUILD_TARGET=aarch64-apple-ios-macabi swift/build_ffi.sh --release --build-std - fi - ) - s.test_spec 'Tests' do |test_spec| test_spec.source_files = 'swift/Tests/*/*.swift' + test_spec.preserve_paths = [ + 'swift/Tests/*/Resources', + ] test_spec.pod_target_xcconfig = { - 'LIBSIGNAL_FFI_LIB_IF_NEEDED' => '', + # Don't also link into the test target. + 'LIBSIGNAL_FFI_LIB_TO_LINK' => '', } end end diff --git a/bin/fetch_archive.py b/bin/fetch_archive.py new file mode 100755 index 00000000..2bfb783c --- /dev/null +++ b/bin/fetch_archive.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# +# Copyright 2023 Signal Messenger, LLC +# SPDX-License-Identifier: AGPL-3.0-only +# + +# This script is based on RingRTC's fetch-artifact.py, +# but simplified for using only as a helper for LibSignalClient.podspec. + +import argparse +import hashlib +import os +import sys +import urllib.request + +from typing import BinaryIO + +UNVERIFIED_DOWNLOAD_NAME = "unverified.tmp" + + +def build_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description='Download and verify a build artifact archive.') + parser.add_argument('-u', '--url', + required=True, + help='URL of an artifact archive') + parser.add_argument('-c', '--checksum', + required=True, + help='sha256sum of the unexpanded artifact archive') + parser.add_argument('-o', '--output-dir', + required=True, + help='Directory to store the archives in') + return parser + + +def download_if_needed(archive_file: str, url: str, checksum: str) -> BinaryIO: + try: + f = open(archive_file, 'rb') + digest = hashlib.sha256() + chunk = f.read1() + while chunk: + digest.update(chunk) + chunk = f.read1() + if digest.hexdigest() == checksum.lower(): + return f + print("existing file '{}' has non-matching checksum {}; re-downloading...".format(archive_file, digest.hexdigest()), file=sys.stderr) + except FileNotFoundError: + pass + + print("downloading {}...".format(archive_file), file=sys.stderr) + try: + with urllib.request.urlopen(url) as response: + digest = hashlib.sha256() + f = open(UNVERIFIED_DOWNLOAD_NAME, 'w+b') + chunk = response.read1() + while chunk: + digest.update(chunk) + f.write(chunk) + chunk = response.read1() + assert digest.hexdigest() == checksum.lower(), "expected {}, actual {}".format(checksum.lower(), digest.hexdigest()) + os.replace(UNVERIFIED_DOWNLOAD_NAME, archive_file) + return f + except urllib.error.HTTPError as e: + print(e, e.filename, file=sys.stderr) + sys.exit(1) + + +def main() -> None: + parser = build_argument_parser() + args = parser.parse_args() + os.makedirs(os.path.abspath(args.output_dir), exist_ok=True) + os.chdir(args.output_dir) + + archive_file = os.path.basename(args.url) + download_if_needed(archive_file, args.url, args.checksum) + + +main() diff --git a/swift/README.md b/swift/README.md index a298ba08..d8fb01e3 100644 --- a/swift/README.md +++ b/swift/README.md @@ -7,15 +7,18 @@ This is a binding to the Signal client code in rust/, implemented on top of the 1. Make sure you are using `use_frameworks!` in your Podfile. LibSignalClient is a Swift pod and as such cannot be compiled as a plain library. -2. Add 'LibSignalClient' and 'SignalCoreKit' as dependencies in your Podfile: +2. Add 'LibSignalClient' and 'SignalCoreKit' as dependencies in your Podfile, as well as the prebuild checksum for the latest release. You can find this in [Signal-iOS's Podfile][]. pod 'LibSignalClient', git: 'https://github.com/signalapp/libsignal.git' pod 'SignalCoreKit', git: 'https://github.com/signalapp/SignalCoreKit.git' + ENV['LIBSIGNAL_FFI_PREBUILD_CHECKSUM'] = '...' 3. Use `pod install` or `pod update` to build the Rust library for all targets. You may be prompted to install Rust dependencies (`cbindgen`, `rust-src`). 4. Build as usual. The Rust library will automatically be linked into the built LibSignalClient.framework. +[Signal-iOS's Podfile]: https://github.com/signalapp/Signal-iOS/blob/main/Podfile + ## Development as a CocoaPod diff --git a/swift/build_ffi.sh b/swift/build_ffi.sh index 67f4abf2..0dcd1976 100755 --- a/swift/build_ffi.sh +++ b/swift/build_ffi.sh @@ -11,6 +11,7 @@ SCRIPT_DIR=$(dirname "$0") cd "${SCRIPT_DIR}"/.. . bin/build_helpers.sh +export IPHONEOS_DEPLOYMENT_TARGET=12.2 export CARGO_PROFILE_RELEASE_DEBUG=1 # enable line tables export CARGO_PROFILE_RELEASE_LTO=fat # use fat LTO to reduce binary size export CFLAGS="-DOPENSSL_SMALL ${CFLAGS:-}" # use small BoringSSL curve tables to reduce binary size