Pods: Change the podspec to download prebuilds

Rather than building the Rust parts of libsignal as part of `pod
install`, fetch them from build-artifacts.signal.org. This requires
adding

    ENV['LIBSIGNAL_FFI_PREBUILD_CHECKSUM'] = '...'

to the consuming Podfile. The referenced archives are downloaded to
~/Library/Caches/org.signal.libsignal, and are unarchived as part of
the build. (The archives are outside the build directory so that a
clean build does not require a new download.)

Building with LibSignalClient as a local pod is still supported; in
that case everything will refer to the local target/ directory
instead. Use swift/build_ffi.sh to build as usual.

.github/workflows/build_and_test.yml

@@ -277,7 +277,18 @@ jobs:
           repository: signalapp/SignalCoreKit
           path: SignalCoreKit
 
-      - run: rustup toolchain install $(cat rust-toolchain) --profile minimal --target x86_64-apple-ios,aarch64-apple-ios,aarch64-apple-ios-sim
+      - run: rustup toolchain install $(cat rust-toolchain) --profile minimal --target x86_64-apple-ios,aarch64-apple-ios-sim
+
+      # Build only the targets that `pod lib lint` will test building.
+      - name: Build for x86_64-apple-ios
+        run: swift/build_ffi.sh --release
+        env:
+          CARGO_BUILD_TARGET: x86_64-apple-ios
+
+      - name: Build for aarch64-apple-ios-sim
+        run: swift/build_ffi.sh --release
+        env:
+          CARGO_BUILD_TARGET: aarch64-apple-ios-sim
 
       - name: Run pod lint
         # No import validation because it tries to build unsupported platforms (like 32-bit iOS).

.github/workflows/lints.yml

@@ -17,6 +17,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - run: pip3 install flake8
+    - run: pip3 install flake8 mypy
     - run: shellcheck **/*.sh
     - run: python3 -m flake8 .
+    # Only include typed Python scripts here.
+    - run: python3 -m mypy bin/fetch_archive.py --python-version 3.6 --strict

.github/workflows/slow_tests.yml

@@ -124,6 +124,31 @@ jobs:
 
       - run: rustup toolchain install $(cat rust-toolchain) --profile minimal --target x86_64-apple-ios,aarch64-apple-ios,aarch64-apple-ios-sim --component rust-src
 
+      - name: Build for x86_64-apple-ios
+        run: swift/build_ffi.sh --release
+        env:
+          CARGO_BUILD_TARGET: x86_64-apple-ios
+
+      - name: Build for aarch64-apple-ios
+        run: swift/build_ffi.sh --release
+        env:
+          CARGO_BUILD_TARGET: aarch64-apple-ios
+
+      - name: Build for aarch64-apple-ios-sim
+        run: swift/build_ffi.sh --release
+        env:
+          CARGO_BUILD_TARGET: aarch64-apple-ios-sim
+
+      - name: Build for x86_64-apple-ios-macabi
+        run: swift/build_ffi.sh --release --build-std
+        env:
+          CARGO_BUILD_TARGET: x86_64-apple-ios-macabi
+
+      - name: Build for aarch64-apple-ios-macabi
+        run: swift/build_ffi.sh --release --build-std
+        env:
+          CARGO_BUILD_TARGET: aarch64-apple-ios-macabi
+
       - name: Run pod lint
         # No import validation because it tries to build unsupported platforms (like 32-bit iOS).
         run: pod lib lint --verbose --platforms=ios --include-podspecs=SignalCoreKit/SignalCoreKit.podspec --skip-import-validation

LibSignalClient.podspec

@@ -14,17 +14,14 @@ Pod::Spec.new do |s|
   s.source           = { :git => 'https://github.com/signalapp/libsignal.git', :tag => "v#{s.version}" }
 
   s.swift_version    = '5'
-  # We use this to set IPHONEOS_DEPLOYMENT_TARGET below.
-  # The Rust compiler driver expects this to always be in the form 'major.minor'.
-  min_deployment_target = '12.2'
-  s.platform         = :ios, min_deployment_target
+  s.platform         = :ios, '12.2'
 
   s.dependency 'SignalCoreKit'
 
   s.source_files = ['swift/Sources/**/*.swift', 'swift/Sources/**/*.m']
   s.preserve_paths = [
-    'target/*/release/libsignal_ffi.a',
     'swift/Sources/SignalFfi',
+    'bin/fetch_archive.py',
   ]
 
   s.pod_target_xcconfig = {
@@ -32,10 +29,17 @@ Pod::Spec.new do |s|
       # Duplicate this here to make sure the search path is passed on to Swift dependencies.
       'SWIFT_INCLUDE_PATHS' => '$(HEADER_SEARCH_PATHS)',
 
+      'LIBSIGNAL_FFI_BUILD_PATH' => 'target/$(CARGO_BUILD_TARGET)/release',
+      # Store libsignal_ffi.a builds in a project-wide directory
+      # because we keep simulator and device builds next to each other.
+      'LIBSIGNAL_FFI_TEMP_DIR' => '$(PROJECT_TEMP_DIR)/libsignal_ffi',
+      'LIBSIGNAL_FFI_LIB_TO_LINK' => '$(LIBSIGNAL_FFI_TEMP_DIR)/$(LIBSIGNAL_FFI_BUILD_PATH)/libsignal_ffi.a',
+
       # Make sure we link the static library, not a dynamic one.
-      # Use an extra level of indirection because CocoaPods messes with OTHER_LDFLAGS too.
-      'LIBSIGNAL_FFI_LIB_IF_NEEDED' => '$(PODS_TARGET_SRCROOT)/target/$(CARGO_BUILD_TARGET)/release/libsignal_ffi.a',
-      'OTHER_LDFLAGS' => '$(LIBSIGNAL_FFI_LIB_IF_NEEDED)',
+      'OTHER_LDFLAGS' => '$(LIBSIGNAL_FFI_LIB_TO_LINK)',
+
+      'LIBSIGNAL_FFI_PREBUILD_ARCHIVE' => "libsignal-client-ios-build-v#{s.version}.tar.gz",
+      'LIBSIGNAL_FFI_PREBUILD_CHECKSUM' => ENV.fetch('LIBSIGNAL_FFI_PREBUILD_CHECKSUM', ''),
 
       'CARGO_BUILD_TARGET[sdk=iphonesimulator*][arch=arm64]' => 'aarch64-apple-ios-sim',
       'CARGO_BUILD_TARGET[sdk=iphonesimulator*][arch=*]' => 'x86_64-apple-ios',
@@ -55,37 +59,47 @@ Pod::Spec.new do |s|
   }
 
   s.script_phases = [
-    { :name => 'Check libsignal-ffi',
-      :execution_position => :before_compile,
-      :script => %q(
-        test -e "${LIBSIGNAL_FFI_LIB_IF_NEEDED}" && exit 0
-        if test -e "${PODS_TARGET_SRCROOT}/swift/build_ffi.sh"; then
-          echo 'error: libsignal_ffi.a not built; run the following to build it:' >&2
-          echo "CARGO_BUILD_TARGET=${CARGO_BUILD_TARGET} \"${PODS_TARGET_SRCROOT}/swift/build_ffi.sh\" --release" >&2
-        else
-          echo 'error: libsignal_ffi.a not built; try re-running `pod install`' >&2
+    { name: 'Download and cache libsignal-ffi',
+      execution_position: :before_compile,
+      script: %q(
+        set -euo pipefail
+        if [ -e "${PODS_TARGET_SRCROOT}/swift/build_ffi.sh" ]; then
+          # Local development
+          exit 0
+        fi
+        "${PODS_TARGET_SRCROOT}"/bin/fetch_archive.py -u "https://build-artifacts.signal.org/libraries/${LIBSIGNAL_FFI_PREBUILD_ARCHIVE}" -c "${LIBSIGNAL_FFI_PREBUILD_CHECKSUM}" -o "${USER_LIBRARY_DIR}/Caches/org.signal.libsignal"
+      ),
+    },
+    { name: 'Extract libsignal-ffi prebuild',
+      execution_position: :before_compile,
+      input_files: ['$(USER_LIBRARY_DIR)/Caches/org.signal.libsignal/$(LIBSIGNAL_FFI_PREBUILD_ARCHIVE)'],
+      output_files: ['$(LIBSIGNAL_FFI_LIB_TO_LINK)'],
+      script: %q(
+        set -euo pipefail
+        rm -rf "${LIBSIGNAL_FFI_TEMP_DIR}"
+        if [ -e "${PODS_TARGET_SRCROOT}/swift/build_ffi.sh" ]; then
+          # Local development
+          ln -fhs "${PODS_TARGET_SRCROOT}" "${LIBSIGNAL_FFI_TEMP_DIR}"
+        elif [ -e "${SCRIPT_INPUT_FILE_0}" ]; then
+          mkdir -p "${LIBSIGNAL_FFI_TEMP_DIR}"
+          cd "${LIBSIGNAL_FFI_TEMP_DIR}"
+          tar --modification-time -x -f "${SCRIPT_INPUT_FILE_0}"
+        else
+          echo 'error: could not download libsignal_ffi.a; please provide LIBSIGNAL_FFI_PREBUILD_CHECKSUM' >&2
+          exit 1
         fi
-        false
       ),
     }
   ]
 
-  s.prepare_command = %Q(
-    set -euo pipefail
-    export IPHONEOS_DEPLOYMENT_TARGET=#{min_deployment_target}
-    CARGO_BUILD_TARGET=aarch64-apple-ios swift/build_ffi.sh --release
-    CARGO_BUILD_TARGET=x86_64-apple-ios swift/build_ffi.sh --release
-    CARGO_BUILD_TARGET=aarch64-apple-ios-sim swift/build_ffi.sh --release
-    if [[ "${SKIP_CATALYST:-0}" != "1" ]]; then
-      CARGO_BUILD_TARGET=x86_64-apple-ios-macabi swift/build_ffi.sh --release --build-std
-      CARGO_BUILD_TARGET=aarch64-apple-ios-macabi swift/build_ffi.sh --release --build-std
-    fi
-  )
-
   s.test_spec 'Tests' do |test_spec|
     test_spec.source_files = 'swift/Tests/*/*.swift'
+    test_spec.preserve_paths = [
+      'swift/Tests/*/Resources',
+    ]
     test_spec.pod_target_xcconfig = {
-      'LIBSIGNAL_FFI_LIB_IF_NEEDED' => '',
+      # Don't also link into the test target.
+      'LIBSIGNAL_FFI_LIB_TO_LINK' => '',
     }
   end
 end

bin/fetch_archive.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+
+#
+# Copyright 2023 Signal Messenger, LLC
+# SPDX-License-Identifier: AGPL-3.0-only
+#
+
+# This script is based on RingRTC's fetch-artifact.py,
+# but simplified for using only as a helper for LibSignalClient.podspec.
+
+import argparse
+import hashlib
+import os
+import sys
+import urllib.request
+
+from typing import BinaryIO
+
+UNVERIFIED_DOWNLOAD_NAME = "unverified.tmp"
+
+
+def build_argument_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description='Download and verify a build artifact archive.')
+    parser.add_argument('-u', '--url',
+                        required=True,
+                        help='URL of an artifact archive')
+    parser.add_argument('-c', '--checksum',
+                        required=True,
+                        help='sha256sum of the unexpanded artifact archive')
+    parser.add_argument('-o', '--output-dir',
+                        required=True,
+                        help='Directory to store the archives in')
+    return parser
+
+
+def download_if_needed(archive_file: str, url: str, checksum: str) -> BinaryIO:
+    try:
+        f = open(archive_file, 'rb')
+        digest = hashlib.sha256()
+        chunk = f.read1()
+        while chunk:
+            digest.update(chunk)
+            chunk = f.read1()
+        if digest.hexdigest() == checksum.lower():
+            return f
+        print("existing file '{}' has non-matching checksum {}; re-downloading...".format(archive_file, digest.hexdigest()), file=sys.stderr)
+    except FileNotFoundError:
+        pass
+
+    print("downloading {}...".format(archive_file), file=sys.stderr)
+    try:
+        with urllib.request.urlopen(url) as response:
+            digest = hashlib.sha256()
+            f = open(UNVERIFIED_DOWNLOAD_NAME, 'w+b')
+            chunk = response.read1()
+            while chunk:
+                digest.update(chunk)
+                f.write(chunk)
+                chunk = response.read1()
+            assert digest.hexdigest() == checksum.lower(), "expected {}, actual {}".format(checksum.lower(), digest.hexdigest())
+            os.replace(UNVERIFIED_DOWNLOAD_NAME, archive_file)
+            return f
+    except urllib.error.HTTPError as e:
+        print(e, e.filename, file=sys.stderr)
+        sys.exit(1)
+
+
+def main() -> None:
+    parser = build_argument_parser()
+    args = parser.parse_args()
+    os.makedirs(os.path.abspath(args.output_dir), exist_ok=True)
+    os.chdir(args.output_dir)
+
+    archive_file = os.path.basename(args.url)
+    download_if_needed(archive_file, args.url, args.checksum)
+
+
+main()

swift/README.md

@@ -7,15 +7,18 @@ This is a binding to the Signal client code in rust/, implemented on top of the
 
 1. Make sure you are using `use_frameworks!` in your Podfile. LibSignalClient is a Swift pod and as such cannot be compiled as a plain library.
 
-2. Add 'LibSignalClient' and 'SignalCoreKit' as dependencies in your Podfile:
+2. Add 'LibSignalClient' and 'SignalCoreKit' as dependencies in your Podfile, as well as the prebuild checksum for the latest release. You can find this in [Signal-iOS's Podfile][].
 
         pod 'LibSignalClient', git: 'https://github.com/signalapp/libsignal.git'
         pod 'SignalCoreKit', git: 'https://github.com/signalapp/SignalCoreKit.git'
+        ENV['LIBSIGNAL_FFI_PREBUILD_CHECKSUM'] = '...'
 
 3. Use `pod install` or `pod update` to build the Rust library for all targets. You may be prompted to install Rust dependencies (`cbindgen`, `rust-src`).
 
 4. Build as usual. The Rust library will automatically be linked into the built LibSignalClient.framework.
 
+[Signal-iOS's Podfile]: https://github.com/signalapp/Signal-iOS/blob/main/Podfile
+
 
 ## Development as a CocoaPod
 

swift/build_ffi.sh

@@ -11,6 +11,7 @@ SCRIPT_DIR=$(dirname "$0")
 cd "${SCRIPT_DIR}"/..
 . bin/build_helpers.sh
 
+export IPHONEOS_DEPLOYMENT_TARGET=12.2
 export CARGO_PROFILE_RELEASE_DEBUG=1 # enable line tables
 export CARGO_PROFILE_RELEASE_LTO=fat # use fat LTO to reduce binary size
 export CFLAGS="-DOPENSSL_SMALL ${CFLAGS:-}" # use small BoringSSL curve tables to reduce binary size