Merge remote-tracking branch 'upstream/dev' into dev

scripts/wk/cfg/ufd.py

@@ -31,6 +31,11 @@ BOOT_FILES = {
   '/boot/grub': 'cfg',
   '/EFI/boot':  'conf',
   }
+IMAGE_BOOT_ENTRIES = {
+  'El Capitan':   'UFD-MACOS-10.11',
+  'High Sierra':  'UFD-MACOS-10.13',
+  'Catalina':     'UFD-MACOS-10.15',
+  }
 
 # Definitions: Sources and Destinations
 ## NOTES: Paths are relative to the root of the ISO/UFD

scripts/wk/kit/ufd.py

@@ -5,14 +5,22 @@ import logging
 import math
 import os
 import shutil
+from subprocess import CalledProcessError
 
 from collections import OrderedDict
 from docopt import docopt
 
 from wk import io, log, std
 from wk.cfg.main import KIT_NAME_FULL, KIT_NAME_SHORT
-from wk.cfg.ufd import BOOT_ENTRIES, BOOT_FILES, ITEMS, ITEMS_HIDDEN, SOURCES
-from wk.exe import run_program
+from wk.cfg.ufd import (
+  BOOT_ENTRIES,
+  BOOT_FILES,
+  IMAGE_BOOT_ENTRIES,
+  ITEMS,
+  ITEMS_HIDDEN,
+  SOURCES,
+  )
+from wk.exe import get_json_from_command, run_program
 from wk.os import linux
 
 
@@ -108,7 +116,7 @@ def build_ufd():
       function=create_table,
       dev_path=ufd_dev,
       use_mbr=args['--use-mbr'],
-      images=args['EXTRA_IMAGES'],
+      images=extra_images,
       )
     try_print.run(
       message='Setting boot flag...',
@@ -174,6 +182,8 @@ def build_ufd():
   try_print.run(
     message='Updating boot entries...',
     function=update_boot_entries,
+    ufd_dev=ufd_dev,
+    images=extra_images,
     )
 
   # Install syslinux (to partition)
@@ -263,7 +273,7 @@ def copy_source(source, items, overwrite=False):
 
 
 def create_table(dev_path, use_mbr=False, images=None):
-  """Create GPT or DOS partition table."""
+  """Create GPT or DOS partition table, returns dict."""
   cmd = [
     'sudo',
     'parted', dev_path,
@@ -488,10 +498,10 @@ def show_selections(args, sources, ufd_dev, ufd_sources, extra_images):
   std.print_standard(' ')
 
 
-def update_boot_entries():
+def update_boot_entries(ufd_dev, images=None):
   """Update boot files for UFD usage"""
   configs = []
-  uuid = get_uuid('/mnt/UFD')
+  uuids = [get_uuid('/mnt/UFD')]
 
   # Find config files
   for c_path, c_ext in BOOT_FILES.items():
@@ -511,8 +521,8 @@ def update_boot_entries():
     '--in-place',
     '--regexp-extended',
     (
-      f's#archisolabel={ISO_LABEL}#archisodevice=/dev/disk/by-uuid/{uuid}#; '
-      f's#by-label/(eSysRescueLiveCD|{ISO_LABEL}|{UFD_LABEL})#by-uuid/{uuid}#'
+      f's#archisolabel={ISO_LABEL}#archisodevice=/dev/disk/by-uuid/{uuids[0]}#; '
+      f's#by-label/(eSysRescueLiveCD|{ISO_LABEL}|{UFD_LABEL})#by-uuid/{uuids[0]}#'
       ),
     *configs,
     ]
@@ -536,6 +546,43 @@ def update_boot_entries():
       ]
     run_program(cmd, check=False)
 
+  # Bail early
+  if not images:
+    return
+
+  # Get PARTUUID values
+  cmd = ['lsblk', '--json', '--output', 'partuuid', ufd_dev]
+  try:
+    uuids = get_json_from_command(cmd)
+  except (CalledProcessError, ValueError):
+    # Bail
+    return
+  uuids = [v['partuuid'] for v in uuids['blockdevices']]
+
+  # Uncomment extra image entries if present
+  for _i, image in enumerate(images):
+    for name, comment in IMAGE_BOOT_ENTRIES.items():
+      if name.lower() in image.name.lower():
+        cmd = [
+          'sudo',
+          'sed',
+          '--in-place',
+          '--regexp-extended',
+          fr's/(#{comment}#.*)"PARTUUID/\1"{uuids[_i+2]}/',
+          *configs,
+          ]
+        run_program(cmd, check=False)
+        cmd = [
+          'sudo',
+          'sed',
+          '--in-place',
+          f's/#{comment}#//',
+          *configs,
+          ]
+        run_program(cmd, check=False)
+        IMAGE_BOOT_ENTRIES.pop(name)
+        break
+
 
 def verify_sources(args, ufd_sources):
   """Check all sources and abort if necessary, returns dict."""

setup/build_linux

@@ -18,6 +18,8 @@ PROFILE_DIR="$BUILD_DIR/archiso-profile"
 REPO_DIR="$BUILD_DIR/repo"
 SKEL_DIR="$PROFILE_DIR/airootfs/etc/skel"
 TEMP_DIR="$BUILD_DIR/temp"
+WORK_DIR="$(mktemp -dt archiso-tmp.XXXXXXXXXX)"
+ISO_DIR="${WORK_DIR}/iso"
 MIRRORLIST_SOURCE='https://archlinux.org/mirrorlist/?country=US&protocol=http&protocol=https&ip_version=4&use_mirror_status=on'
 if command -v nano >/dev/null 2>&1; then
   EDITOR=nano
@@ -121,15 +123,14 @@ function update_live_env() {
   sed -i -r "s/SUPPORT_URL/$KIT_NAME_SHORT/" "$PROFILE_DIR/profiledef.sh"
 
   # Boot config (legacy)
-  mkdir -p "$PROFILE_DIR/arch"
+  mkdir -p "$PROFILE_DIR/syslinux/wimboot"
   cp "$ROOT_DIR/images/Syslinux.jpg" "$PROFILE_DIR/syslinux/syslinux.jpg"
   sed -i -r "s/__+/$KIT_NAME_FULL/" "$PROFILE_DIR/syslinux/syslinux.cfg"
   mkdir -p "$TEMP_DIR" 2>/dev/null
   curl -Lo "$TEMP_DIR/wimboot.zip" "http://git.ipxe.org/releases/wimboot/wimboot-latest.zip"
-  7z e -aoa "$TEMP_DIR/wimboot.zip" -o"$PROFILE_DIR/arch/boot" 'wimboot*/LICENSE.txt' 'wimboot*/README.txt' 'wimboot*/wimboot'
+  7z e -aoa "$TEMP_DIR/wimboot.zip" -o"$PROFILE_DIR/syslinux/wimboot" 'wimboot*/LICENSE.txt' 'wimboot*/README.txt' 'wimboot*/wimboot'
 
   # Boot config (UEFI)
-  mkdir -p "$PROFILE_DIR/EFI/boot"
   cp  "/usr/share/refind/refind_x64.efi" "$PROFILE_DIR/EFI/boot/bootx64.efi"
   cp "$ROOT_DIR/images/rEFInd.png" "$PROFILE_DIR/EFI/boot/rEFInd.png"
   rsync -aI "/usr/share/refind/drivers_x64/" "$PROFILE_DIR/EFI/boot/drivers_x64/"
@@ -170,6 +171,9 @@ function update_live_env() {
   # MOTD
   sed -i -r "s/_+/$KIT_NAME_FULL Linux Environment/" "$PROFILE_DIR/airootfs/etc/motd"
 
+  # Network
+  ln -s "/run/systemd/resolve/stub-resolv.conf" "$PROFILE_DIR/airootfs/etc/resolv.conf"
+
   # Oh My ZSH
   git clone --depth=1 https://github.com/robbyrussell/oh-my-zsh.git "$SKEL_DIR/.oh-my-zsh"
   rm -Rf "$SKEL_DIR/.oh-my-zsh/.git"
@@ -206,7 +210,7 @@ function update_live_env() {
   # Tech user
   echo "tech:$(echo "$TECH_PASSWORD" | openssl passwd -6 -stdin):14871::::::" >> "$PROFILE_DIR/airootfs/etc/shadow"
 
-  # Timezonew
+  # Timezone
   ln -sf "/usr/share/zoneinfo/$LINUX_TIME_ZONE" "$PROFILE_DIR/airootfs/etc/localtime"
 
   if [[ "${1:-}" != "--minimal" ]]; then
@@ -338,16 +342,29 @@ function build_iso() {
   # Build ISO
   prefix="${KIT_NAME_SHORT}-Linux"
   label="${KIT_NAME_SHORT}_LINUX"
-  #mkarchiso -w "$BUILD_DIR/work" -o "$OUT_DIR" -v "$PROFILE_DIR" | tee -a "$LOG_DIR/$DATETIME.log"
-  mkarchiso -w /tmp/archiso-tmp -o "$OUT_DIR" -v "$PROFILE_DIR" | tee -a "$LOG_DIR/$DATETIME.log"
+  mkarchiso \
+    -o "$OUT_DIR" \
+    -w ${WORK_DIR} \
+    -v "$PROFILE_DIR" \
+    | tee -a "$LOG_DIR/$DATETIME.log"
+
+  # Build better ISO
+  rm -r "${ISO_DIR:-safety}/EFI"
+  rm -r "${ISO_DIR:-safety}/loader"
+  rsync -aI "$PROFILE_DIR/EFI/" "${ISO_DIR:-safety}/EFI/"
+  rsync -aI --ignore-existing "$PROFILE_DIR/syslinux/" "${ISO_DIR:-safety}/syslinux/"
+  ## Sketchy bit ##
+  . /usr/bin/mkarchiso -o "${OUT_DIR}" -w "${WORK_DIR}" "${PROFILE_DIR}"
+  isofs_dir="${ISO_DIR}"
+  image_name="${KIT_NAME_SHORT}-Linux-${DATE}-x86_64.iso"
+  rm "${OUT_DIR}/${image_name}"
+  _build_iso_image
 
   # Cleanup
   echo "Removing temp files..."
   rm "$TEMP_DIR/Linux" -Rf | tee -a "$LOG_DIR/$DATETIME.log"
-  #sudo umount -R "$BUILD_DIR/work" || true
-  #sudo rm -rf "$BUILD_DIR/work"
-  sudo umount -R /tmp/archiso-tmp || true
-  sudo rm -rf /tmp/archiso-tmp
+  sudo umount -R ${WORK_DIR} || true
+  sudo rm -rf ${WORK_DIR}
 
   echo "Reverting permissions..."
   chown $REAL_USER:$REAL_USER "$PROFILE_DIR" -R

setup/linux/packages/base

@@ -48,6 +48,7 @@ mkinitcpio-archiso
 mprime
 nano
 ncdu
+openssh
 p7zip
 perl
 pipes.sh

setup/linux/packages/dependencies

@@ -3,6 +3,7 @@ attr
 base-devel
 curl
 dos2unix
+erofs-utils
 git
 gtk-doc
 gtk3

setup/linux/profile_base/EFI/boot/refind.conf

@@ -17,7 +17,7 @@ hideui arrows,badges
 # Entries
 menuentry "MemTest86" {
     icon /EFI/boot/icons/wk_memtest.png
-    loader /EFI/memtest86/memtestx64.efi
+    loader /EFI/memtest86/bootx64.efi
 }
 
 menuentry "Linux" {
@@ -41,6 +41,36 @@ menuentry "Linux" {
     #UFD-MINIMAL#}
 }
 
+#UFD-MACOS-10.11#menuentry "macOS (El Capitan)" {
+#UFD-MACOS-10.11#    icon /EFI/boot/icons/wk_mac_10.11.png
+#UFD-MACOS-10.11#    volume "PARTUUID"
+#UFD-MACOS-10.11#    loader /System/Library/CoreServices/secretboot.efi
+#UFD-MACOS-10.11#    graphics
+#UFD-MACOS-10.11#    submenuentry "Mac MemTest" {
+#UFD-MACOS-10.11#        add_options "-s"
+#UFD-MACOS-10.11#    }
+#UFD-MACOS-10.11#}
+
+#UFD-MACOS-10.13#menuentry "macOS (High Sierra)" {
+#UFD-MACOS-10.13#    icon /EFI/boot/icons/wk_mac_10.13.png
+#UFD-MACOS-10.13#    volume "PARTUUID"
+#UFD-MACOS-10.13#    loader /System/Library/CoreServices/secretboot.efi
+#UFD-MACOS-10.13#    graphics
+#UFD-MACOS-10.13#    submenuentry "Mac MemTest" {
+#UFD-MACOS-10.13#        add_options "-s"
+#UFD-MACOS-10.13#    }
+#UFD-MACOS-10.13#}
+
+#UFD-MACOS-10.15#menuentry "macOS (Catalina)" {
+#UFD-MACOS-10.15#    icon /EFI/boot/icons/wk_mac_10.15.png
+#UFD-MACOS-10.15#    volume "PARTUUID"
+#UFD-MACOS-10.15#    loader /System/Library/CoreServices/secretboot.efi
+#UFD-MACOS-10.15#    graphics
+#UFD-MACOS-10.15#    submenuentry "Mac MemTest" {
+#UFD-MACOS-10.15#        add_options "-s"
+#UFD-MACOS-10.15#    }
+#UFD-MACOS-10.15#}
+
 #UFD-WINPE#menuentry "WindowsPE" {
 #UFD-WINPE#    ostype windows
 #UFD-WINPE#    icon /EFI/boot/icons/wk_win.png

setup/linux/profile_base/airootfs/etc/modprobe.d/broadcom-wl.conf

@@ -0,0 +1,7 @@
+# The broadcom-wl package requires some modules to be disabled in order to use
+# wl. Since the ISO image needs to cover many hardware cases, this file
+# overrides the default blacklist in /usr/lib/modprobe.d/
+#
+# If you need to use wl, you may need to delete this file, then `rmmod` any
+# already-loaded modules that are now blacklisted before proceeding to modprobe
+# wl itself.

setup/linux/profile_base/airootfs/etc/pacman.d/hooks/40-locale-gen.hook

@@ -0,0 +1,13 @@
+# remove from airootfs!
+[Trigger]
+Operation = Install
+Type = Package
+Target = glibc
+
+[Action]
+Description = Uncommenting en_US.UTF-8 locale and running locale-gen...
+When = PostTransaction
+Depends = glibc
+Depends = sed
+Depends = sh
+Exec = /bin/sh -c "sed -i 's/#\(en_US\.UTF-8\)/\1/' /etc/locale.gen && locale-gen"

setup/linux/profile_base/airootfs/etc/pacman.d/hooks/zzzz99-remove-custom-hooks-from-airootfs.hook

@@ -0,0 +1,18 @@
+# remove from airootfs!
+# As a workaround for https://bugs.archlinux.org/task/49347 , remove pacman hooks specific to the ISO build process.
+# If not, they would be used when pacstrap is run in the live environment.
+
+[Trigger]
+Operation = Install
+Operation = Upgrade
+Operation = Remove
+Type = Package
+Target = *
+
+[Action]
+Description = Work around FS#49347 by removing custom pacman hooks that are only required during ISO build...
+When = PostTransaction
+Depends = sh
+Depends = coreutils
+Depends = grep
+Exec = /bin/sh -c "rm -- $(grep -Frl 'remove from airootfs' /etc/pacman.d/hooks/)"

setup/linux/profile_base/airootfs/etc/skel/.update_network

@@ -2,24 +2,42 @@
 #
 ## Setup network and update hostname
 
-# Wait for WiFi
-echo -n "Waiting for network... "
-sleep 3s
-echo "Done"
+IP=
+NEW_HOSTNAME=
 
-# Set hostname
-echo -n "Updating hostname... "
+# Load test station details if present
+if [[ -s "/run/archiso/bootmnt/teststation.name" ]]; then
+  NEW_HOSTNAME="$(head -1 "/run/archiso/bootmnt/teststation.name" \
+    | sed -r 's/\s//g')"
+fi
+if [[ -s "/run/archiso/bootmnt/teststation.wall" ]]; then
+  rm "${HOME}/.wallpaper" >/dev/null 2>&1
+  ln -s "/run/archiso/bootmnt/teststation.wall" "${HOME}/.wallpaper"
+fi
+
+# Wait for network connection and get IP
+echo -n "Waiting for network... "
+for x in {1..3}; do
+  sleep 1s
   IP="$(ip a show scope global \
     | grep inet \
     | head -1 \
-  | sed -r 's#.*inet ([0-9]+.[0-9]+.[0-9]+.[0-9]+.)/.*#\1#')"
+    | sed -r 's#.*inet ([0-9]+.[0-9]+.[0-9]+.[0-9]+)/.*#\1#')"
   if [[ "${IP:+x}" ]]; then
+    break
+  fi
+done
+echo "Done"
+
+# Set hostname
+if [[ -z "${NEW_HOSTNAME:+x}" && "${IP:+x}" ]]; then
   NEW_HOSTNAME="$(dig +noall +answer +short -x "$IP" \
     | grep -v ';' \
     | head -1 \
     | sed 's/\.$//')"
 fi
 if [[ "${NEW_HOSTNAME:+x}" ]]; then
+  echo -n "Updating hostname... "
   sudo hostnamectl set-hostname "${NEW_HOSTNAME}"
 fi
 echo "Done"

setup/linux/profile_base/airootfs/etc/systemd/journald.conf.d/volatile-storage.conf

@@ -1,5 +1,2 @@
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
 [Journal]
 Storage=volatile

setup/linux/profile_base/airootfs/etc/systemd/logind.conf.d/do-not-suspend.conf

@@ -1,6 +1,3 @@
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
 [Login]
 HandleSuspendKey=ignore
 HandleHibernateKey=ignore

setup/linux/profile_base/airootfs/etc/systemd/network/20-ethernet.network

@@ -1,6 +1,3 @@
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
 [Match]
 Name=en*
 Name=eth*

setup/linux/profile_base/airootfs/etc/systemd/network/20-wireless.network

@@ -1,6 +1,3 @@
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
 [Match]
 Name=wlp*
 Name=wlan*

setup/linux/profile_base/airootfs/etc/systemd/system/etc-pacman.d-gnupg.mount

@@ -1,6 +1,3 @@
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
 [Unit]
 Description=Temporary /etc/pacman.d/gnupg directory
 

setup/linux/profile_base/airootfs/etc/systemd/system/pacman-init.service

@@ -1,6 +1,3 @@
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
 [Unit]
 Description=Initializes Pacman keyring
 Wants=haveged.service

setup/linux/profile_base/airootfs/root/customize_airootfs.sh

@@ -1,12 +0,0 @@
-#!/bin/env bash
-#
-# Warning: customize_airootfs.sh is deprecated! Support for it will be removed in a future archiso version.
-
-set -o errexit
-set -o errtrace
-set -o nounset
-set -o pipefail
-
-sed -i 's/#\(en_US\.UTF-8\)/\1/' /etc/locale.gen
-locale-gen
-

setup/linux/profile_base/efiboot/loader/entries/01-archiso-x86_64-linux.conf

@@ -1,6 +1,3 @@
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
 title   Arch Linux (x86_64, UEFI)
 linux   /%INSTALL_DIR%/boot/x86_64/vmlinuz-linux
 initrd  /%INSTALL_DIR%/boot/x86_64/initramfs-linux.img

setup/linux/profile_base/efiboot/loader/entries/02-archiso-x86_64-ram-linux.conf

@@ -0,0 +1,4 @@
+title   Arch Linux (x86_64, UEFI) Copy to RAM
+linux   /%INSTALL_DIR%/boot/x86_64/vmlinuz-linux
+initrd  /%INSTALL_DIR%/boot/x86_64/initramfs-linux.img
+options archisobasedir=%INSTALL_DIR% archisolabel=%ARCHISO_LABEL% copytoram

setup/linux/profile_base/efiboot/loader/loader.conf

@@ -1,5 +1,2 @@
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
 timeout 3
-default archiso-x86_64-linux.conf
+default 01-archiso-x86_64-linux.conf

setup/linux/profile_base/pacman.conf

@@ -1,6 +1,7 @@
 [options]
 HoldPkg = pacman glibc
 Architecture = auto
+ParallelDownloads = 5
 SigLevel = Required DatabaseOptional
 LocalFileSigLevel = Optional
 

setup/linux/profile_base/profiledef.sh

@@ -7,6 +7,7 @@ iso_publisher="SUPPORT_URL"
 iso_application="KIT_NAME_FULL Linux Environment"
 iso_version="$(date +%Y-%m-%d)"
 install_dir="arch"
+buildmodes=('iso')
 bootmodes=('bios.syslinux.mbr' 'bios.syslinux.eltorito' 'uefi-x64.systemd-boot.esp' 'uefi-x64.systemd-boot.eltorito')
 arch="x86_64"
 pacman_conf="pacman.conf"

setup/linux/profile_base/syslinux/syslinux.cfg

@@ -1,4 +1,4 @@
-SERIAL 0 38400
+SERIAL 0 115200
 UI vesamenu.c32
 MENU TITLE _______
 MENU BACKGROUND syslinux.jpg

setup/linux/profile_base/syslinux/winpe.cfg

@@ -5,4 +5,4 @@ A live Windows environment
 ENDTEXT
 MENU LABEL Windows PE
 COM32 linux.c32
-APPEND wimboot gui initrdfile=/sources/BOOTMGR,/sources/BCD,/sources/boot.sdi,/sources/boot.wim
+APPEND wimboot/wimboot gui initrdfile=/sources/BOOTMGR,/sources/BCD,/sources/boot.sdi,/sources/boot.wim