backups: add shared message backup test cases

.github/workflows/build_and_test.yml

@@ -34,6 +34,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0
       id: filter
@@ -108,11 +110,13 @@ jobs:
           toolchain: "$(cat rust-toolchain)"
         - version: stable
           toolchain: "${LIBSIGNAL_MINIMUM_SUPPORTED_RUST_VERSION}"
-    
+
     timeout-minutes: 45
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - run: sudo apt-get update && sudo apt-get install protobuf-compiler
 
@@ -234,6 +238,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - run: sudo apt-get update && sudo apt-get install gcc-multilib g++-multilib protobuf-compiler
 
@@ -264,6 +270,7 @@ jobs:
 
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       with:
+        submodules: recursive
         # Download all commits so we can search for the merge base with origin/main.
         fetch-depth: 0
 
@@ -306,6 +313,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - run: rustup toolchain install $(cat rust-toolchain) --profile minimal
 
@@ -362,6 +371,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - run: rustup toolchain install $(cat rust-toolchain) --profile minimal
 
@@ -397,6 +408,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          submodules: recursive
 
       - run: brew install protobuf swiftlint
 

.github/workflows/ios_artifacts.yml

@@ -28,6 +28,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - name: Checking run eligibility
       uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1

.github/workflows/jni_artifacts.yml

@@ -34,6 +34,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - name: Checking run eligibility
       uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
@@ -96,6 +98,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - run: rustup toolchain install $(cat rust-toolchain) --profile minimal
 
@@ -115,6 +119,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - name: Download built libraries
       id: download

.github/workflows/lints.yml

@@ -17,6 +17,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
     - run: pip3 install flake8 mypy
     - run: shellcheck **/*.sh bin/verify_duplicate_crates bin/adb-run-test
     - run: python3 -m flake8 .

.github/workflows/npm.yml

@@ -38,6 +38,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - name: Checking run eligibility
       uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
@@ -104,6 +106,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - run: node/docker-prebuildify.sh
 
@@ -121,6 +125,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - run: rustup toolchain install $(cat rust-toolchain) --profile minimal
 
@@ -140,6 +146,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
       with:

.github/workflows/slow_tests.yml

@@ -22,6 +22,8 @@ jobs:
       has-changes: ${{ steps.check.outputs.has-changes }}
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
     - run: git log --after '24 hours ago' --exit-code || echo 'has-changes=true' >> $GITHUB_OUTPUT
       id: check
 
@@ -34,6 +36,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
     - run: make -C java java_test
     - name: Upload JNI libraries
       uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
@@ -66,6 +70,8 @@ jobs:
             sudo udevadm trigger --name-match=kvm
 
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          submodules: recursive
 
       - name: Download JNI libraries
         id: download
@@ -113,6 +119,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
     - run: node/docker-prebuildify.sh
     - run: yarn tsc && yarn test
       working-directory: node
@@ -128,6 +136,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
     - run: rustup toolchain install $(cat rust-toolchain) --profile minimal --target aarch64-pc-windows-msvc
     # install nasm compiler for boring
     - name: Install nasm
@@ -160,6 +170,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          submodules: recursive
 
       - run: rustup toolchain install $(cat rust-toolchain) --profile minimal --target x86_64-apple-ios,aarch64-apple-ios,aarch64-apple-ios-sim --component rust-src
 
@@ -192,6 +204,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        submodules: recursive
 
     - run: sudo apt-get update && sudo apt-get install gcc-multilib g++-multilib protobuf-compiler
 

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "rust/message-backup/tests/res/Signal-Message-Backup-Tests"]
+	path = rust/message-backup/tests/res/Signal-Message-Backup-Tests
+	url = https://github.com/signalapp/Signal-Message-Backup-Tests.git

java/client/src/test/resources/org/signal/libsignal/messagebackup/encryptedbackup.binproto.encrypted

@@ -1 +1 @@
-../../../../../../../../../rust/message-backup/tests/res/test-cases/valid/new_account.binproto.encrypted
+../../../../../../../../../rust/message-backup/tests/res/test-cases/valid-encrypted/new-account.binproto.encrypted

node/ts/test/new_account.binproto.encrypted

@@ -1 +1 @@
-../../../rust/message-backup/tests/res/test-cases/valid/new_account.binproto.encrypted
+../../../rust/message-backup/tests/res/test-cases/valid-encrypted/new-account.binproto.encrypted

rust/message-backup/tests/res/Signal-Message-Backup-Tests

@@ -0,0 +1 @@
+Subproject commit a3c81678572f4e1963f05aa5be1d5f9baa6b15fa

rust/message-backup/tests/res/test-cases/valid

@@ -0,0 +1 @@
+../Signal-Message-Backup-Tests/test-cases/

rust/message-backup/tests/res/test-cases/valid/new-account.jsonproto

@@ -1,32 +0,0 @@
-[
-  {
-    "version": "1",
-    "backupTimeMs": "1705692409729"
-  },
-  {
-    "recipient": {
-      "id": "1",
-      "contact": {
-        "aci": "IiMfftM7TXC2Brinq8Wbvw==",
-        "pni": "kgNIuOIxTxmzIYUp0E+4Og==",
-        "e164": "18045550102",
-        "registered": "REGISTERED",
-        "profileKey": "a9y+KGrHvMM94LKU4dMlFVjFa36zyjxsCv1vcgvjsOs=",
-        "profileGivenName": "Alex iOS test",
-        "profileFamilyName": "Konradi"
-      }
-    }
-  },
-  {
-    "recipient": {
-      "id": "2",
-      "self": {}
-    }
-  },
-  {
-    "chat": {
-      "id": "1",
-      "recipientId": "2"
-    }
-  }
-]

rust/message-backup/tests/test_cases.rs

@@ -21,32 +21,30 @@ const BACKUP_PURPOSE: Purpose = Purpose::RemoteBackup;
 #[dir_test(
         dir: "$CARGO_MANIFEST_DIR/tests/res/test-cases",
         glob: "valid/*.jsonproto",
-        postfix: "binproto"
+        postfix: "jsonproto"
     )]
 fn is_valid_json_proto(input: Fixture<&str>) {
     let json_contents = input.into_content();
     let json_contents = serde_json::from_str(json_contents).expect("invalid JSON");
     let json_array = assert_matches!(json_contents, serde_json::Value::Array(contents) => contents);
-    let binary =
+    let binproto =
         libsignal_message_backup::backup::convert_from_json(json_array).expect("failed to convert");
-
-    // Check via the library interface.
-    let input = Cursor::new(&*binary);
-    let reader = BackupReader::new_unencrypted(input, BACKUP_PURPOSE);
-    validate(reader);
-
-    // The CLI tool should agree.
-    validator_command()
-        .arg("-")
-        .args(["--purpose", BACKUP_PURPOSE.into()])
-        .write_stdin(binary)
-        .ok()
-        .expect("command failed");
+    validate_proto(&binproto)
 }
 
 #[dir_test(
         dir: "$CARGO_MANIFEST_DIR/tests/res/test-cases",
-        glob: "valid/*.binproto.encrypted",
+        glob: "valid/*.binproto",
+        postfix: "binproto"
+        loader: read_file
+    )]
+fn is_valid_binary_proto(input: Fixture<Vec<u8>>) {
+    validate_proto(input.content())
+}
+
+#[dir_test(
+        dir: "$CARGO_MANIFEST_DIR/tests/res/test-cases",
+        glob: "valid-encrypted/*.binproto.encrypted",
         loader: PathBuf::from,
         postfix: "encrypted"
     )]
@@ -98,10 +96,10 @@ fn invalid_jsonproto(input: Fixture<PathBuf>) {
         serde_json::from_str(&std::fs::read_to_string(path).expect("failed to read"))
             .expect("invalid JSON");
     let json_array = assert_matches!(json_contents, serde_json::Value::Array(contents) => contents);
-    let binary =
+    let binproto =
         libsignal_message_backup::backup::convert_from_json(json_array).expect("failed to convert");
 
-    let input = Cursor::new(&*binary);
+    let input = Cursor::new(&*binproto);
     let reader = BackupReader::new_unencrypted(input, Purpose::RemoteBackup);
 
     let ReadResult {
@@ -123,10 +121,29 @@ fn invalid_jsonproto(input: Fixture<PathBuf>) {
     assert_eq!(text, expected_text);
 }
 
+fn read_file(path: &str) -> Vec<u8> {
+    std::fs::read(path).expect("can read")
+}
+
 fn write_expected_error() -> bool {
     std::env::var_os("OVERWRITE_EXPECTED_OUTPUT").is_some()
 }
 
+fn validate_proto(binproto: &[u8]) {
+    // Check via the library interface.
+    let input = Cursor::new(binproto);
+    let reader = BackupReader::new_unencrypted(input, BACKUP_PURPOSE);
+    validate(reader);
+
+    // The CLI tool should agree.
+    validator_command()
+        .arg("-")
+        .args(["--purpose", BACKUP_PURPOSE.into()])
+        .write_stdin(binproto)
+        .ok()
+        .expect("command failed");
+}
+
 fn validate(mut reader: BackupReader<impl AsyncRead + Unpin + VerifyHmac>) {
     reader.visitor = |msg| println!("{msg:#?}");
 

swift/Tests/LibSignalClientTests/Resources/new_account.binproto.encrypted

@@ -1 +1 @@
-../../../../rust/message-backup/tests/res/test-cases/valid/new_account.binproto.encrypted
+../../../../rust/message-backup/tests/res/test-cases/valid-encrypted/new-account.binproto.encrypted