Run "softwareupdate" on vanilla images (#204)

* Run "softwareupdate" on vanilla images

* Install Command Line Tools for Xcode

* Run Ansible after password-less sudo is configured

* Don't use SFTP

* Fix Ansible playbook path

* No updates are available → No new software available

* stderr_lines → stderr

* Dynamically resolve MACOS_NUMBER

.ci/cirrus.base.yml

@@ -9,6 +9,8 @@ task:
   <<: *defaults
   pull_vanilla_script:
     - tart pull ghcr.io/cirruslabs/macos-$MACOS_VERSION-vanilla:latest
+  install_sshpass_script:
+    - brew install sshpass
   build_base_script:
     - packer init templates/base.pkr.hcl
     - packer build -var macos_version="$MACOS_VERSION" templates/base.pkr.hcl

.ci/cirrus.vanilla.yml

@@ -1,15 +1,16 @@
+env:
+  RESOLVE_VM_BASE_NAME: "ghcr.io/cirruslabs/macos-${MACOS_VERSION}-vanilla:latest"
+  RESOLVE_VM_NAME: "resolve-macos-number-task-id-${CIRRUS_TASK_ID}"
+  RESOLVE_FILE: "${RESOLVE_VM_NAME}.txt"
+
 task:
-  name: "Update Vanilla Image ($MACOS_VERSION $MACOS_NUMBER)"
+  name: "Update Vanilla Image ($MACOS_VERSION)"
   env:
     matrix:
       - MACOS_VERSION: sequoia
-        MACOS_NUMBER: 15.3
       - MACOS_VERSION: sonoma
-        MACOS_NUMBER: 14.6
       - MACOS_VERSION: ventura
-        MACOS_NUMBER: 13.6
       - MACOS_VERSION: monterey
-        MACOS_NUMBER: 12.6.1
   only_if: $CIRRUS_BRANCH == $CIRRUS_DEFAULT_BRANCH && changesInclude("templates/vanilla-$MACOS_VERSION.pkr.hcl")
   <<: *defaults
   build_script:
@@ -17,6 +18,11 @@ task:
     - packer build templates/vanilla-$MACOS_VERSION.pkr.hcl
   disable_sip_script:
     - packer build -var vm_name=$MACOS_VERSION-vanilla templates/disable-sip.pkr.hcl
+  resolve_macos_number_script:
+    - packer build -var vm_base_name=$RESOLVE_VM_BASE_NAME -var vm_name=$RESOLVE_VM_NAME -var resolve_file=$RESOLVE_FILE templates/resolve-macos-number.pkr.hcl
+    - echo "MACOS_NUMBER=$(cat $RESOLVE_FILE)" >> $CIRRUS_ENV
+    - rm $RESOLVE_FILE
+    - tart delete $RESOLVE_VM_NAME
   push_script:
     - tart push $MACOS_VERSION-vanilla ghcr.io/cirruslabs/macos-$MACOS_VERSION-vanilla:latest ghcr.io/cirruslabs/macos-$MACOS_VERSION-vanilla:$MACOS_NUMBER
   always:

ansible/playbook-system-updater.yml

@@ -0,0 +1,5 @@
+- hosts: default
+  roles:
+    - system-updater
+  vars:
+    ansible_password: admin

ansible/roles/system-updater/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: Perform first "softwareupdate" invocation
+  include_tasks: softwareupdate.yml
+
+# Needed after a major macOS update, otherwise things like
+# Command Line Tools won't be updated
+- name: Perform second "softwareupdate" invocation
+  include_tasks: softwareupdate.yml
+
+# This one looks weird, but unfortunately there's no other way around, because Homebrew
+# is not designed to run as root (see https://gist.github.com/irazasyed/7732946
+# for more details).
+- name: fix up /usr/local permissions for Homebrew
+  file:
+    path: /usr/local/share/man
+    state: directory
+    owner: "{{ ansible_user_id }}"
+    recurse: yes
+  become: yes
+
+- name: Ensure that there are no more software updates available (1/2)
+  command: "softwareupdate --all --list"
+  register: check_updates_result
+
+- name: Ensure that there are no more software updates available (2/2)
+  assert:
+    that:
+      - "'No new software available' in check_updates_result.stderr"

ansible/roles/system-updater/tasks/softwareupdate.yml

@@ -0,0 +1,26 @@
+# It seems that we must always pass "--restart" command-line argument to "softwareupdate",
+# otherwise on the OS update the "softwareupdate" will be stuck at "Downloaded: macOS [...]"
+- name: install all macOS updates
+  command:
+    cmd: "softwareupdate --all --install --agree-to-license --force --restart --user admin --stdinpass"
+    stdin: "{{ ansible_password }}"
+  register: update_result
+  # Work around the following:
+  # > Data could not be sent to remote host [...].
+  # > Make sure this host can be reached over ssh:
+  # > ssh: connect to host [...] port 22: Connection refused.
+  ignore_unreachable: yes
+  # Ignore SIGTERM/SIGKILL sent "softwareupdate" process
+  # when the system reboots due to --restart
+  failed_when: update_result.rc not in [0, 9, -9, 15, -15]
+  become: yes
+
+# Wait for the connection since the previous command could restart the host
+- name: wait for connection
+  wait_for_connection:
+    # We need to wait long enough for the "softwareupdate" to initiate the reboot,
+    # otherwise it's possible that we'll interrupt the process by running
+    # the commands below on a non-restarted system.
+    delay: 60
+    timeout: 1800
+  when: "'No updates are available' not in (update_result.stderr_lines | join('\n'))"

templates/resolve-macos-number.pkr.hcl

@@ -0,0 +1,46 @@
+packer {
+  required_plugins {
+    tart = {
+      version = ">= 1.12.0"
+      source  = "github.com/cirruslabs/tart"
+    }
+  }
+}
+
+variable "vm_base_name" {
+  type = string
+}
+
+variable "vm_name" {
+  type = string
+}
+
+variable "resolve_file" {
+  type = string
+}
+
+source "tart-cli" "tart" {
+  vm_base_name = "${var.vm_base_name}"
+  vm_name      = "${var.vm_name}"
+  cpu_count    = 4
+  memory_gb    = 8
+  ssh_password = "admin"
+  ssh_username = "admin"
+  ssh_timeout  = "120s"
+}
+
+build {
+  sources = ["source.tart-cli.tart"]
+
+  provisioner "shell" {
+    inline = [
+      "sw_vers --productVersion > /tmp/sw-vers-product-version.txt",
+    ]
+  }
+
+  provisioner "file" {
+    source = "/tmp/sw-vers-product-version.txt"
+    destination = "${var.resolve_file}"
+    direction = "download"
+  }
+}

templates/vanilla-monterey.pkr.hcl

@@ -4,6 +4,10 @@ packer {
       version = ">= 1.2.0"
       source  = "github.com/cirruslabs/tart"
     }
+    ansible = {
+      version = "~> 1"
+      source = "github.com/hashicorp/ansible"
+    }
   }
 }
 
@@ -79,6 +83,9 @@ source "tart-cli" "tart" {
   // A (hopefully) temporary workaround for Virtualization.Framework's
   // installation process not fully finishing in a timely manner
   create_grace_time = "30s"
+
+  // Keep the recovery partition, otherwise it's not possible to "softwareupdate"
+  recovery_partition = "keep"
 }
 
 build {
@@ -116,4 +123,17 @@ build {
       "sysadminctl -screenLock off -password admin",
     ]
   }
+
+  provisioner "shell" {
+    inline = [
+      # Install command-line tools
+      "touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress",
+      "softwareupdate --list | sed -n 's/.*Label: \\(Command Line Tools for Xcode-.*\\)/\\1/p' | tr '\\n' '\\0' | xargs -0 softwareupdate --install",
+      "rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress",
+    ]
+  }
+
+  provisioner "ansible" {
+    playbook_file = "ansible/playbook-system-updater.yml"
+  }
 }

templates/vanilla-sequoia.pkr.hcl

@@ -4,6 +4,10 @@ packer {
       version = ">= 1.12.0"
       source  = "github.com/cirruslabs/tart"
     }
+    ansible = {
+      version = "~> 1"
+      source = "github.com/hashicorp/ansible"
+    }
   }
 }
 
@@ -83,6 +87,9 @@ source "tart-cli" "tart" {
   // A (hopefully) temporary workaround for Virtualization.Framework's
   // installation process not fully finishing in a timely manner
   create_grace_time = "30s"
+
+  // Keep the recovery partition, otherwise it's not possible to "softwareupdate"
+  recovery_partition = "keep"
 }
 
 build {
@@ -118,4 +125,17 @@ build {
       "sysadminctl -screenLock off -password admin",
     ]
   }
+
+  provisioner "shell" {
+    inline = [
+      # Install command-line tools
+      "touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress",
+      "softwareupdate --list | sed -n 's/.*Label: \\(Command Line Tools for Xcode-.*\\)/\\1/p' | tr '\\n' '\\0' | xargs -0 softwareupdate --install",
+      "rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress",
+    ]
+  }
+
+  provisioner "ansible" {
+    playbook_file = "ansible/playbook-system-updater.yml"
+  }
 }

templates/vanilla-sonoma.pkr.hcl

@@ -4,6 +4,10 @@ packer {
       version = ">= 1.12.0"
       source  = "github.com/cirruslabs/tart"
     }
+    ansible = {
+      version = "~> 1"
+      source = "github.com/hashicorp/ansible"
+    }
   }
 }
 
@@ -78,6 +82,9 @@ source "tart-cli" "tart" {
   // A (hopefully) temporary workaround for Virtualization.Framework's
   // installation process not fully finishing in a timely manner
   create_grace_time = "30s"
+
+  // Keep the recovery partition, otherwise it's not possible to "softwareupdate"
+  recovery_partition = "keep"
 }
 
 build {
@@ -115,4 +122,17 @@ build {
       "sysadminctl -screenLock off -password admin",
     ]
   }
+
+  provisioner "shell" {
+    inline = [
+      # Install command-line tools
+      "touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress",
+      "softwareupdate --list | sed -n 's/.*Label: \\(Command Line Tools for Xcode-.*\\)/\\1/p' | tr '\\n' '\\0' | xargs -0 softwareupdate --install",
+      "rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress",
+    ]
+  }
+
+  provisioner "ansible" {
+    playbook_file = "ansible/playbook-system-updater.yml"
+  }
 }

templates/vanilla-ventura.pkr.hcl

@@ -4,6 +4,10 @@ packer {
       version = ">= 1.12.0"
       source  = "github.com/cirruslabs/tart"
     }
+    ansible = {
+      version = "~> 1"
+      source = "github.com/hashicorp/ansible"
+    }
   }
 }
 
@@ -86,6 +90,9 @@ source "tart-cli" "tart" {
   // A (hopefully) temporary workaround for Virtualization.Framework's
   // installation process not fully finishing in a timely manner
   create_grace_time = "30s"
+
+  // Keep the recovery partition, otherwise it's not possible to "softwareupdate"
+  recovery_partition = "keep"
 }
 
 build {
@@ -124,4 +131,17 @@ build {
       "defaults -currentHost write com.apple.screensaver idleTime 0"
     ]
   }
+
+  provisioner "shell" {
+    inline = [
+      # Install command-line tools
+      "touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress",
+      "softwareupdate --list | sed -n 's/.*Label: \\(Command Line Tools for Xcode-.*\\)/\\1/p' | tr '\\n' '\\0' | xargs -0 softwareupdate --install",
+      "rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress",
+    ]
+  }
+
+  provisioner "ansible" {
+    playbook_file = "ansible/playbook-system-updater.yml"
+  }
 }