From 685f897d307646c7e3e67c641b7424d930636eee Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 08:10:57 +0100 Subject: [PATCH 01/43] Add new_build.sh --- src/new_build.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/new_build.sh diff --git a/src/new_build.sh b/src/new_build.sh new file mode 100644 index 0000000..86e393e --- /dev/null +++ b/src/new_build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Slimmed-down Docker build script +# Copyright (c) 2017 Julian Xhokaxhiu +# Copyright (C) 2017-2018 Nicola Corna +# Copyright (C) 2024 Pete Fotheringham +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see From cada06e48e3ba3af4e9bbbaf4153e26799bafef8 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 08:17:59 +0100 Subject: [PATCH 02/43] For 21.0 builds, call new_build.sh --- src/init.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/init.sh b/src/init.sh index b45ff5c..bcc63ae 100755 --- a/src/init.sh +++ b/src/init.sh @@ -65,14 +65,19 @@ visibility = ["//visibility:public"], ) _EOB +build_file="build.sh" +if [ "$BRANCH_NAME" = "lineage-21.0" ]; then + build_file="new_build.sh" +fi + if [ "$CRONTAB_TIME" = "now" ]; then - /root/build.sh + /root/$build_file else # Initialize the cronjob cronFile=/tmp/buildcron printf "SHELL=/bin/bash\n" > $cronFile printenv -0 | sed -e 's/=\x0/=""\n/g' | sed -e 's/\x0/\n/g' | sed -e "s/_=/PRINTENV=/g" >> $cronFile - printf '\n%s /usr/bin/flock -n /var/lock/build.lock /root/build.sh >> /var/log/docker.log 2>&1\n' "$CRONTAB_TIME" >> $cronFile + printf "\n%s /usr/bin/flock -n /var/lock/build.lock /root/$build_file >> /var/log/docker.log 2>&1\n" "$CRONTAB_TIME" >> $cronFile crontab $cronFile rm $cronFile From b76678c67bc1994f05607ee12825b1ad79f32bfb Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 11:36:26 +0100 Subject: [PATCH 03/43] Add outline documentation --- src/new_build.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index 86e393e..66dbdfc 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -17,3 +17,31 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see + + +# Outline +# - Call `begin.sh` +# - Handle parameters and environment variables +# - CLEAN_OUTDIR +# - PARALLEL_JOBS +# - RETRY_FETCHES +# - handle manifests +# - Sync mirror +# - Branch-specific stuff +# - main sync and build loop For each device in `$DEVICE_LIST` +# - setup build overlay +# - `repo init` +# - `repo sync` +# - Call `before.sh` +# - `breakfast` - in case of failure, call +# - `post-build.sh` +# - `do_cleanup` +# - `mka` +# - move artefacts to `ZIPDIR` +# - ROM zip file +# - `.img` files +# - create the checksum files +# - Remove old zips and logs +# - call `post-build.sh` +# - call `do_cleanup` +# - call `end.sh` From 8b67dbd91f210f8379ee32bfef4c1aa9318f8a11 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 11:39:24 +0100 Subject: [PATCH 04/43] Add `do_cleanup` function --- src/new_build.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index 66dbdfc..ef6e327 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -45,3 +45,34 @@ # - call `post-build.sh` # - call `do_cleanup` # - call `end.sh` + +# do_cleanup function +do_cleanup() { + echo ">> [$(date)] Cleaning up" | tee -a "$DEBUG_LOG" + + if [ "$BUILD_OVERLAY" = true ]; then + # The Jack server must be stopped manually, as we want to unmount $TMP_DIR/merged + cd "$TMP_DIR" + if [ -f "$TMP_DIR/merged/prebuilts/sdk/tools/jack-admin" ]; then + "$TMP_DIR/merged/prebuilts/sdk/tools/jack-admin kill-server" &> /dev/null || true + fi + lsof | grep "$TMP_DIR/merged" | awk '{ print $2 }' | sort -u | xargs -r kill &> /dev/null || true + + while lsof | grep -q "$TMP_DIR"/merged; do + sleep 1 + done + + umount "$TMP_DIR/merged" + fi + + if [ "$CLEAN_AFTER_BUILD" = true ]; then + echo ">> [$(date)] Cleaning source dir for device $codename" | tee -a "$DEBUG_LOG" + if [ "$BUILD_OVERLAY" = true ]; then + cd "$TMP_DIR" + rm -rf ./* || true + else + cd "$source_dir" + (set +eu ; mka "${jobs_arg[@]}" clean) &>> "$DEBUG_LOG" + fi + fi +} From e71904ff3bc0375399f0175e576fe1c5bba1ed15 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 11:44:57 +0100 Subject: [PATCH 05/43] Add call to begin.sh --- src/new_build.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index ef6e327..be0e0a0 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -76,3 +76,17 @@ do_cleanup() { fi fi } + +# Build script + +set -eEuo pipefail +repo_log="$LOGS_DIR/repo-$(date +%Y%m%d).log" + +# cd to working directory +cd "$SRC_DIR" + +# Call `begin.sh` +if [ -f /root/userscripts/begin.sh ]; then + echo ">> [$(date)] Running begin.sh" + /root/userscripts/begin.sh || { echo ">> [$(date)] Error: begin.sh failed!"; exit 1; } +fi From 3e3f2365e57cf58feff95c9c87a1d01698eae281 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 11:49:45 +0100 Subject: [PATCH 06/43] Handle CLEAN_OUTDIR --- src/new_build.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index be0e0a0..0366388 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -90,3 +90,14 @@ if [ -f /root/userscripts/begin.sh ]; then echo ">> [$(date)] Running begin.sh" /root/userscripts/begin.sh || { echo ">> [$(date)] Error: begin.sh failed!"; exit 1; } fi + +# Handle parameters and environment variables +## CLEAN_OUTDIR +if [ "$CLEAN_OUTDIR" = true ]; then + echo ">> [$(date)] Cleaning '$ZIP_DIR'" + rm -rf "${ZIP_DIR:?}/"* +fi + +## PARALLEL_JOBS + +## RETRY_FETCHES From e4a99b43c0c32e1eeef4d24f980b37d028d622c0 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 11:51:16 +0100 Subject: [PATCH 07/43] Handle PARALLEL_JOBS and RETRY_FETCHES --- src/new_build.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index 0366388..1219b22 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -99,5 +99,23 @@ if [ "$CLEAN_OUTDIR" = true ]; then fi ## PARALLEL_JOBS +jobs_arg=() +if [ -n "${PARALLEL_JOBS-}" ]; then + if [[ "$PARALLEL_JOBS" =~ ^[1-9][0-9]*$ ]]; then + jobs_arg+=( "-j$PARALLEL_JOBS" ) + else + echo "PARALLEL_JOBS is not a positive number: $PARALLEL_JOBS" + exit 1 + fi +fi ## RETRY_FETCHES +retry_fetches_arg=() +if [ -n "${RETRY_FETCHES-}" ]; then + if [[ "$RETRY_FETCHES" =~ ^[1-9][0-9]*$ ]]; then + retry_fetches_arg+=( "--retry-fetches=$RETRY_FETCHES" ) + else + echo "RETRY_FETCHES is not a positive number: $RETRY_FETCHES" + exit 1 + fi +fi From 0e99bf648616bbc1a947a386b58c9242ce40d945 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 12:49:35 +0100 Subject: [PATCH 08/43] Handle local manifests --- src/new_build.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index 1219b22..15b3e37 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -25,7 +25,7 @@ # - CLEAN_OUTDIR # - PARALLEL_JOBS # - RETRY_FETCHES -# - handle manifests +# - handle local manifests # - Sync mirror # - Branch-specific stuff # - main sync and build loop For each device in `$DEVICE_LIST` @@ -119,3 +119,17 @@ if [ -n "${RETRY_FETCHES-}" ]; then exit 1 fi fi + +# Handle local manifests +## Copy local manifests +echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'" +mkdir -p .repo/local_manifests +rsync -a --delete --include '*.xml' --exclude '*' "$LMANIFEST_DIR/" .repo/local_manifests/ + +## Pick up TheMuppets manifest if required +rm -f .repo/local_manifests/proprietary.xml +if [ "$INCLUDE_PROPRIETARY" = true ]; then + wget -q -O .repo/local_manifests/proprietary.xml "https://raw.githubusercontent.com/TheMuppets/manifests/$themuppets_branch/muppets.xml" + /root/build_manifest.py --remote "https://gitlab.com" --remotename "gitlab_https" \ + "https://gitlab.com/the-muppets/manifest/raw/$themuppets_branch/muppets.xml" .repo/local_manifests/proprietary_gitlab.xml +fi From d202e8ec05a74879b77887d4f59ab9583af21127 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 13:03:30 +0100 Subject: [PATCH 09/43] Init & sync the mirror --- src/new_build.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index 15b3e37..e2d3a00 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -26,7 +26,7 @@ # - PARALLEL_JOBS # - RETRY_FETCHES # - handle local manifests -# - Sync mirror +# - Sync mirror if we're using one # - Branch-specific stuff # - main sync and build loop For each device in `$DEVICE_LIST` # - setup build overlay @@ -133,3 +133,23 @@ if [ "$INCLUDE_PROPRIETARY" = true ]; then /root/build_manifest.py --remote "https://gitlab.com" --remotename "gitlab_https" \ "https://gitlab.com/the-muppets/manifest/raw/$themuppets_branch/muppets.xml" .repo/local_manifests/proprietary_gitlab.xml fi + +# Sync mirror if we're using one +if [ "$LOCAL_MIRROR" = true ]; then + + cd "$MIRROR_DIR" + if [ "$INIT_MIRROR" = true ]; then + if [ ! -d .repo ]; then + echo ">> [$(date)] Initializing mirror repository" | tee -a "$repo_log" + ( yes||: ) | repo init -u https://github.com/LineageOS/mirror --mirror --no-clone-bundle -p linux --git-lfs &>> "$repo_log" + fi + else + echo ">> [$(date)] Initializing mirror repository disabled" | tee -a "$repo_log" + fi + if [ "$SYNC_MIRROR" = true ]; then + echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log" + repo sync "${jobs_arg[@]}" --force-sync --no-clone-bundle &>> "$repo_log" + else + echo ">> [$(date)] Sync mirror repository disabled" | tee -a "$repo_log" + fi +fi From b053978b3853e03038648e75576d8097bf24c910 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 13:05:20 +0100 Subject: [PATCH 10/43] Branch-specific stuff --- src/new_build.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index e2d3a00..3aea0e0 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -153,3 +153,31 @@ if [ "$LOCAL_MIRROR" = true ]; then echo ">> [$(date)] Sync mirror repository disabled" | tee -a "$repo_log" fi fi + +# Branch-specific stuff +branch=$BRANCH_NAME +branch_dir=${branch//[^[:alnum:]]/_} +branch_dir=${branch_dir^^} +echo ">> [$(date)] Branch: $branch" + +vendor=lineage + +devices=$DEVICE_LIST +echo ">> [$(date)] Devices: $devices" + +if [ -n "$branch" ] && [ -n "$devices" ]; then + case "$branch" in + lineage-21.0*) + themuppets_branch="lineage-21.0" + android_version="14" + ;; + *) + echo ">> [$(date)] Building branch $branch is not (yet) suppported" + exit 1 + ;; + esac + android_version_major=$(cut -d '.' -f 1 <<< $android_version) + + mkdir -p "$SRC_DIR/$branch_dir" + cd "$SRC_DIR/$branch_dir" +fi From 2515eebd4e4430a78992f4c3c31e58ed5fe56c51 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 13:08:02 +0100 Subject: [PATCH 11/43] Handle BRANCH_NAME and DEVICE_LIST earlier --- src/new_build.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index 3aea0e0..0f4f751 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -92,6 +92,14 @@ if [ -f /root/userscripts/begin.sh ]; then fi # Handle parameters and environment variables +branch=$BRANCH_NAME +echo ">> [$(date)] Branch: $branch" + +devices=$DEVICE_LIST +echo ">> [$(date)] Devices: $devices" + +vendor=lineage + ## CLEAN_OUTDIR if [ "$CLEAN_OUTDIR" = true ]; then echo ">> [$(date)] Cleaning '$ZIP_DIR'" @@ -155,15 +163,8 @@ if [ "$LOCAL_MIRROR" = true ]; then fi # Branch-specific stuff -branch=$BRANCH_NAME branch_dir=${branch//[^[:alnum:]]/_} branch_dir=${branch_dir^^} -echo ">> [$(date)] Branch: $branch" - -vendor=lineage - -devices=$DEVICE_LIST -echo ">> [$(date)] Devices: $devices" if [ -n "$branch" ] && [ -n "$devices" ]; then case "$branch" in From 048dcb788484ff52f7ab339346de169b2ab5d884 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 13:27:43 +0100 Subject: [PATCH 12/43] Add main per-device loop --- src/new_build.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index 0f4f751..ed20881 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -28,7 +28,8 @@ # - handle local manifests # - Sync mirror if we're using one # - Branch-specific stuff -# - main sync and build loop For each device in `$DEVICE_LIST` +# - main sync and build loop +# For each device in `$DEVICE_LIST` # - setup build overlay # - `repo init` # - `repo sync` @@ -182,3 +183,17 @@ if [ -n "$branch" ] && [ -n "$devices" ]; then mkdir -p "$SRC_DIR/$branch_dir" cd "$SRC_DIR/$branch_dir" fi + +# - main sync and build loop +# For each device in `$DEVICE_LIST` +for codename in ${devices//,/ }; do + if [ -n "$codename" ]; then + # - `repo init` + # - `repo sync` + # - setup our overlays + + fi + +# More stuff to do + +done From 0e081d03ec6dd77e56e8f0eb84abdb8aadba6791 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 13:38:30 +0100 Subject: [PATCH 13/43] Setup overlays --- src/new_build.sh | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index ed20881..eb154f6 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -188,9 +188,32 @@ fi # For each device in `$DEVICE_LIST` for codename in ${devices//,/ }; do if [ -n "$codename" ]; then + builddate=$(date +%Y%m%d) # - `repo init` # - `repo sync` - # - setup our overlays + # Setup our overlays + if [ "$BUILD_OVERLAY" = true ]; then + lowerdir=$SRC_DIR/$branch_dir + upperdir=$TMP_DIR/device + workdir=$TMP_DIR/workdir + merged=$TMP_DIR/merged + mkdir -p "$upperdir" "$workdir" "$merged" + mount -t overlay overlay -o lowerdir="$lowerdir",upperdir="$upperdir",workdir="$workdir" "$merged" + source_dir="$TMP_DIR/merged" + else + source_dir="$SRC_DIR/$branch_dir" + fi + + mkdir -p "vendor/$vendor/overlay/microg/" + sed -i "1s;^;PRODUCT_PACKAGE_OVERLAYS := vendor/$vendor/overlay/microg\n;" "vendor/$vendor/config/common.mk" + + makefile_containing_version="vendor/$vendor/config/common.mk" + if [ -f "vendor/$vendor/config/version.mk" ]; then + makefile_containing_version="vendor/$vendor/config/version.mk" + fi + los_ver_major=$(sed -n -e 's/^\s*PRODUCT_VERSION_MAJOR = //p' "$makefile_containing_version") + los_ver_minor=$(sed -n -e 's/^\s*PRODUCT_VERSION_MINOR = //p' "$makefile_containing_version") + los_ver="$los_ver_major.$los_ver_minor" fi From 87964b2ad2dbd16b8bfb4e7ccd578919893df927 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 13:44:21 +0100 Subject: [PATCH 14/43] Setup subdirectories --- src/new_build.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index eb154f6..d03dacc 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -30,6 +30,7 @@ # - Branch-specific stuff # - main sync and build loop # For each device in `$DEVICE_LIST` +# - setup subdirectories # - setup build overlay # - `repo init` # - `repo sync` @@ -189,6 +190,22 @@ fi for codename in ${devices//,/ }; do if [ -n "$codename" ]; then builddate=$(date +%Y%m%d) + + # setup subdirectories + if [ "$ZIP_SUBDIR" = true ]; then + zipsubdir=$codename + mkdir -p "$ZIP_DIR/$zipsubdir" + else + zipsubdir= + fi + if [ "$LOGS_SUBDIR" = true ]; then + logsubdir=$codename + mkdir -p "$LOGS_DIR/$logsubdir" + else + logsubdir= + fi + DEBUG_LOG="$LOGS_DIR/$logsubdir/lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename.log" + # - `repo init` # - `repo sync` # Setup our overlays @@ -214,7 +231,7 @@ for codename in ${devices//,/ }; do los_ver_major=$(sed -n -e 's/^\s*PRODUCT_VERSION_MAJOR = //p' "$makefile_containing_version") los_ver_minor=$(sed -n -e 's/^\s*PRODUCT_VERSION_MINOR = //p' "$makefile_containing_version") los_ver="$los_ver_major.$los_ver_minor" - + fi # More stuff to do From b8b65c657b37216b0f2accc630164559e2a4773d Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 13:54:23 +0100 Subject: [PATCH 15/43] Add repo init --- src/new_build.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index d03dacc..07e15f4 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -207,6 +207,18 @@ for codename in ${devices//,/ }; do DEBUG_LOG="$LOGS_DIR/$logsubdir/lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename.log" # - `repo init` + # ToDo: do we need to add REPO_VERSION - see https://github.com/lineageos-infra/build-config/commit/312e3242d04db35945ce815ab35864a86b14b866 + if [ "$CALL_REPO_INIT" = true ]; then + echo ">> [$(date)] (Re)initializing branch repository" | tee -a "$repo_log" + if [ "$LOCAL_MIRROR" = true ]; then + ( yes||: ) | repo init -u https://github.com/LineageOS/android.git --reference "$MIRROR_DIR" -b "$branch" -g default,-darwin,-muppets,muppets_${DEVICE} --git-lfs &>> "$repo_log" + else + ( yes||: ) | repo init -u https://github.com/LineageOS/android.git -b "$branch" -g default,-darwin,-muppets,muppets_${DEVICE} --git-lfs &>> "$repo_log" + fi + else + echo ">> [$(date)] Calling repo init disabled" + fi + # - `repo sync` # Setup our overlays if [ "$BUILD_OVERLAY" = true ]; then From 571d8d45672dde8d84c871dc26e7f02afa6a9a86 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 13:56:41 +0100 Subject: [PATCH 16/43] Add repo sync --- src/new_build.sh | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index 07e15f4..c267258 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -206,7 +206,7 @@ for codename in ${devices//,/ }; do fi DEBUG_LOG="$LOGS_DIR/$logsubdir/lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename.log" - # - `repo init` + # `repo init` # ToDo: do we need to add REPO_VERSION - see https://github.com/lineageos-infra/build-config/commit/312e3242d04db35945ce815ab35864a86b14b866 if [ "$CALL_REPO_INIT" = true ]; then echo ">> [$(date)] (Re)initializing branch repository" | tee -a "$repo_log" @@ -219,7 +219,26 @@ for codename in ${devices//,/ }; do echo ">> [$(date)] Calling repo init disabled" fi - # - `repo sync` + # `repo sync` + if [ "$CALL_REPO_SYNC" = true ]; then + echo ">> [$(date)] Syncing branch repository" | tee -a "$repo_log" + repo sync "${jobs_arg[@]}" -c --force-sync &>> "$repo_log" + else + echo ">> [$(date)] Syncing branch repository disabled" | tee -a "$repo_log" + fi + + if [ "$CALL_GIT_LFS_PULL" = true ]; then + echo ">> [$(date)] Calling git lfs pull" | tee -a "$repo_log" + repo forall -v -c git lfs pull &>> "$repo_log" + else + echo ">> [$(date)] Calling git lfs pull disabled" | tee -a "$repo_log" + fi + + if [ ! -d "vendor/$vendor" ]; then + echo ">> [$(date)] Missing \"vendor/$vendor\", aborting" + exit 1 + fi + # Setup our overlays if [ "$BUILD_OVERLAY" = true ]; then lowerdir=$SRC_DIR/$branch_dir From f5df79adc27b619cfd6b96e4eac071eb5e30148a Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 14:09:51 +0100 Subject: [PATCH 17/43] Custom packages, keys and environment setup --- src/new_build.sh | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index c267258..8785f2b 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -31,13 +31,17 @@ # - main sync and build loop # For each device in `$DEVICE_LIST` # - setup subdirectories -# - setup build overlay +# - setup our overlays +# - Add custom packages to be installed +# - Handle keys +# - Prepare the environment # - `repo init` # - `repo sync` # - Call `before.sh` # - `breakfast` - in case of failure, call # - `post-build.sh` # - `do_cleanup` +# - Call `pre-build.sh` # - `mka` # - move artefacts to `ZIPDIR` # - ROM zip file @@ -238,7 +242,7 @@ for codename in ${devices//,/ }; do echo ">> [$(date)] Missing \"vendor/$vendor\", aborting" exit 1 fi - + # Setup our overlays if [ "$BUILD_OVERLAY" = true ]; then lowerdir=$SRC_DIR/$branch_dir @@ -263,6 +267,38 @@ for codename in ${devices//,/ }; do los_ver_minor=$(sed -n -e 's/^\s*PRODUCT_VERSION_MINOR = //p' "$makefile_containing_version") los_ver="$los_ver_major.$los_ver_minor" + # Add custom packages to be installed + if [ -n "$CUSTOM_PACKAGES" ]; then + echo ">> [$(date)] Adding custom packages ($CUSTOM_PACKAGES)" + sed -i "1s;^;PRODUCT_PACKAGES += $CUSTOM_PACKAGES\n\n;" "vendor/$vendor/config/common.mk" + fi + + # Handle keys + if [ "$SIGN_BUILDS" = true ]; then + echo ">> [$(date)] Adding keys path ($KEYS_DIR)" + # Soong (Android 9+) complains if the signing keys are outside the build path + ln -sf "$KEYS_DIR" user-keys + if [ "$android_version_major" -lt "10" ]; then + sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" + fi + + if [ "$android_version_major" -ge "10" ]; then + sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := user-keys/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := user-keys/releasekey\n\n;" "vendor/$vendor/config/common.mk" + fi + fi + + # Prepare the environment + if [ "$PREPARE_BUILD_ENVIRONMENT" = true ]; then + echo ">> [$(date)] Preparing build environment" + set +eu + # shellcheck source=/dev/null + source build/envsetup.sh > /dev/null + set -eu + else + echo ">> [$(date)] Preparing build environment disabled" + fi + + fi # More stuff to do From bc6051d96250981045e79e0217ab4aef7a4283df Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 15:03:00 +0100 Subject: [PATCH 18/43] Call before.sh (and correct doc comment) --- src/new_build.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index 8785f2b..70e83f0 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -31,12 +31,12 @@ # - main sync and build loop # For each device in `$DEVICE_LIST` # - setup subdirectories +# - `repo init` +# - `repo sync` # - setup our overlays # - Add custom packages to be installed # - Handle keys # - Prepare the environment -# - `repo init` -# - `repo sync` # - Call `before.sh` # - `breakfast` - in case of failure, call # - `post-build.sh` @@ -298,6 +298,14 @@ for codename in ${devices//,/ }; do echo ">> [$(date)] Preparing build environment disabled" fi + # Call `before.sh` + if [ -f /root/userscripts/before.sh ]; then + echo ">> [$(date)] Running before.sh" + echo "before.sh is now called *after* repo sync." + echo "In previous versions, iot was called *before* repo sync" + /root/userscripts/before.sh || { echo ">> [$(date)] Error: before.sh failed for $branch!"; userscriptfail=true; continue; } + fi + fi From f2ceed579c1748fb2b4e54cb04da4d7791a78e39 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 15:05:15 +0100 Subject: [PATCH 19/43] Call breakfast --- src/new_build.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index 70e83f0..d29d9ac 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -306,6 +306,27 @@ for codename in ${devices//,/ }; do /root/userscripts/before.sh || { echo ">> [$(date)] Error: before.sh failed for $branch!"; userscriptfail=true; continue; } fi + # Call breakfast + breakfast_returncode=0 + if [ "$CALL_BREAKFAST" = true ]; then + set +eu + breakfast "$codename" "$BUILD_TYPE" &>> "$DEBUG_LOG" + breakfast_returncode=$? + set -eu + else + echo ">> [$(date)] Calling breakfast disabled" + fi + + if [ $breakfast_returncode -ne 0 ]; then + echo ">> [$(date)] breakfast failed for $codename, $branch branch" | tee -a "$DEBUG_LOG" + # call post-build.sh so the failure is logged in a way that is more visible + if [ -f /root/userscripts/post-build.sh ]; then + echo ">> [$(date)] Running post-build.sh for $codename" >> "$DEBUG_LOG" + /root/userscripts/post-build.sh "$codename" false "$branch" &>> "$DEBUG_LOG" || echo ">> [$(date)] Warning: post-build.sh failed!" + fi + do_cleanup + continue + fi fi From 264eee5c3622ecf11e197ddf9235dcd10de4f72a Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 15:06:46 +0100 Subject: [PATCH 20/43] Call pre-build.sh --- src/new_build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index d29d9ac..3f3f842 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -328,6 +328,13 @@ for codename in ${devices//,/ }; do continue fi + # Call pre-build.sh + if [ -f /root/userscripts/pre-build.sh ]; then + echo ">> [$(date)] Running pre-build.sh for $codename" >> "$DEBUG_LOG" + /root/userscripts/pre-build.sh "$codename" &>> "$DEBUG_LOG" || { + echo ">> [$(date)] Error: pre-build.sh failed for $codename on $branch!"; userscriptfail=true; continue; } + fi + fi # More stuff to do From 3f1fdc1928b837e0c78a97720eb8f9d412101cd1 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 15:23:46 +0100 Subject: [PATCH 21/43] Call mka and handle the outcome --- src/new_build.sh | 52 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index 3f3f842..ee9d000 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -42,7 +42,7 @@ # - `post-build.sh` # - `do_cleanup` # - Call `pre-build.sh` -# - `mka` +# - Call `mka` # - move artefacts to `ZIPDIR` # - ROM zip file # - `.img` files @@ -335,6 +335,56 @@ for codename in ${devices//,/ }; do echo ">> [$(date)] Error: pre-build.sh failed for $codename on $branch!"; userscriptfail=true; continue; } fi + # Call mka + build_successful=true + if [ "$CALL_MKA" = true ]; then + # Start the build + echo ">> [$(date)] Starting build for $codename, $branch branch" | tee -a "$DEBUG_LOG" + build_successful=false + files_to_hash=() + + if (set +eu ; mka "${jobs_arg[@]}" target-files-package bacon) &>> "$DEBUG_LOG"; then + echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a "$DEBUG_LOG" + + # Move the ROM zip files to the main OUT directory + cd out/target/product/"$codename" + files_to_hash=() + for build in lineage-*.zip; do + cp -v system/build.prop "$ZIP_DIR/$zipsubdir/$build.prop" &>> "$DEBUG_LOG" + mv "$build" "$ZIP_DIR/$zipsubdir/" &>> "$DEBUG_LOG" + files_to_hash+=( "$build" ) + done + + # Now handle the .img files - where are they? + img_dir=$(find "$source_dir/out/target/product/$codename/obj/PACKAGING" -name "IMAGES") + if [ -d "$img_dir" ]; then + cd "$img_dir" + fi + + # rename and copy the images to the zips directory + for image in recovery boot vendor_boot dtbo super_empty vbmeta vendor_kernel_boot init_boot; do + if [ -f "$image.img" ]; then + recovery_name="lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename-$image.img" + echo ">> [$(date)] Copying $image.img" to "$ZIP_DIR/$zipsubdir/$recovery_name" >> "$DEBUG_LOG" + cp "$image.img" "$ZIP_DIR/$zipsubdir/$recovery_name" &>> "$DEBUG_LOG" + files_to_hash+=( "$recovery_name" ) + fi + done + + # create the checksum files + cd "$ZIP_DIR/$zipsubdir" + for f in "${files_to_hash[@]}"; do + sha256sum "$f" > "$ZIP_DIR/$zipsubdir/$f.sha256sum" + done + cd "$source_dir" + build_successful=true + else + echo ">> [$(date)] Failed build for $codename" | tee -a "$DEBUG_LOG" + fi + else + echo ">> [$(date)] Calling mka for $codename, $branch branch disabled" + fi + fi # More stuff to do From 926866cb3243aef0cb045935086b45c7163c6d2f Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 15:27:46 +0100 Subject: [PATCH 22/43] Remove old zips and logs --- src/new_build.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index ee9d000..9e00a3e 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -381,6 +381,22 @@ for codename in ${devices//,/ }; do else echo ">> [$(date)] Failed build for $codename" | tee -a "$DEBUG_LOG" fi + + # Remove old zips and logs + if [ "$DELETE_OLD_ZIPS" -gt "0" ]; then + if [ "$ZIP_SUBDIR" = true ]; then + /usr/bin/python /root/clean_up.py -n "$DELETE_OLD_ZIPS" -V "$los_ver" -N 1 "$ZIP_DIR/$zipsubdir" + else + /usr/bin/python /root/clean_up.py -n "$DELETE_OLD_ZIPS" -V "$los_ver" -N 1 -c "$codename" "$ZIP_DIR" + fi + fi + if [ "$DELETE_OLD_LOGS" -gt "0" ]; then + if [ "$LOGS_SUBDIR" = true ]; then + /usr/bin/python /root/clean_up.py -n "$DELETE_OLD_LOGS" -V "$los_ver" -N 1 "$LOGS_DIR/$logsubdir" + else + /usr/bin/python /root/clean_up.py -n "$DELETE_OLD_LOGS" -V "$los_ver" -N 1 -c "$codename" "$LOGS_DIR" + fi + fi else echo ">> [$(date)] Calling mka for $codename, $branch branch disabled" fi From 1f7c41500f996b51a90085d795f12d858666aefd Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 15:39:14 +0100 Subject: [PATCH 23/43] Call post-build.sh and clean up --- src/new_build.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index 9e00a3e..44b26cd 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -401,8 +401,13 @@ for codename in ${devices//,/ }; do echo ">> [$(date)] Calling mka for $codename, $branch branch disabled" fi + # call post-build.sh + if [ -f /root/userscripts/post-build.sh ]; then + echo ">> [$(date)] Running post-build.sh for $codename" >> "$DEBUG_LOG" + /root/userscripts/post-build.sh "$codename" "$build_successful" "$branch" &>> "$DEBUG_LOG" || { + echo ">> [$(date)] Error: post-build.sh failed for $codename on $branch!"; userscriptfail=true; } + fi + echo ">> [$(date)] Finishing build for $codename" | tee -a "$DEBUG_LOG" + do_cleanup fi - -# More stuff to do - done From 5b66b04504e10899284a0a2271f8e37e174263bc Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 15:46:58 +0100 Subject: [PATCH 24/43] Fix shellcheck errors --- src/new_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index 44b26cd..7716247 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -215,9 +215,9 @@ for codename in ${devices//,/ }; do if [ "$CALL_REPO_INIT" = true ]; then echo ">> [$(date)] (Re)initializing branch repository" | tee -a "$repo_log" if [ "$LOCAL_MIRROR" = true ]; then - ( yes||: ) | repo init -u https://github.com/LineageOS/android.git --reference "$MIRROR_DIR" -b "$branch" -g default,-darwin,-muppets,muppets_${DEVICE} --git-lfs &>> "$repo_log" + ( yes||: ) | repo init -u https://github.com/LineageOS/android.git --reference "$MIRROR_DIR" -b "$branch" -g default,-darwin,-muppets,muppets_"${DEVICE}" --git-lfs &>> "$repo_log" else - ( yes||: ) | repo init -u https://github.com/LineageOS/android.git -b "$branch" -g default,-darwin,-muppets,muppets_${DEVICE} --git-lfs &>> "$repo_log" + ( yes||: ) | repo init -u https://github.com/LineageOS/android.git -b "$branch" -g default,-darwin,-muppets,muppets_"${DEVICE}" --git-lfs &>> "$repo_log" fi else echo ">> [$(date)] Calling repo init disabled" From eb1b7fa10726691cbd796c997b8f34a4efb27e4e Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 16:00:14 +0100 Subject: [PATCH 25/43] Check userscriptfail *for each device*. Exit immediately if true --- src/new_build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index 7716247..09fd4c6 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -408,6 +408,11 @@ for codename in ${devices//,/ }; do echo ">> [$(date)] Error: post-build.sh failed for $codename on $branch!"; userscriptfail=true; } fi echo ">> [$(date)] Finishing build for $codename" | tee -a "$DEBUG_LOG" + do_cleanup + if [ $userscriptfail = true ]; then + echo ">> [$(date)] One or more userscripts failed!" + exit 1 + fi fi done From 800cfad229f755da463d306093bbbd724d369274 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 16:03:49 +0100 Subject: [PATCH 26/43] Call end.sh --- src/new_build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index 09fd4c6..81cc3b9 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -416,3 +416,8 @@ for codename in ${devices//,/ }; do fi fi done + +if [ -f /root/userscripts/end.sh ]; then + echo ">> [$(date)] Running end.sh" + /root/userscripts/end.sh || echo ">> [$(date)] Error: end.sh failed!" +fi From c4fe3d74d36525474b179d70084cc0ffb5414914 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 16:05:07 +0100 Subject: [PATCH 27/43] Another shellcheck error --- src/new_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index 81cc3b9..627a26d 100644 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -410,7 +410,7 @@ for codename in ${devices//,/ }; do echo ">> [$(date)] Finishing build for $codename" | tee -a "$DEBUG_LOG" do_cleanup - if [ $userscriptfail = true ]; then + if [ "$userscriptfail" = true ]; then echo ">> [$(date)] One or more userscripts failed!" exit 1 fi From 77f209cd0db4c24c65c82c6bd8c344934d63dd33 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 17:38:23 +0100 Subject: [PATCH 28/43] new_build.sh must be executable --- src/new_build.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/new_build.sh diff --git a/src/new_build.sh b/src/new_build.sh old mode 100644 new mode 100755 From bbceafb66b17ced9a8f8f856cb03727bb675b5f9 Mon Sep 17 00:00:00 2001 From: Pete Date: Mon, 2 Sep 2024 17:50:56 +0100 Subject: [PATCH 29/43] MOve branch specific stuff before handling manifests --- src/new_build.sh | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index 627a26d..ed83d08 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -25,9 +25,9 @@ # - CLEAN_OUTDIR # - PARALLEL_JOBS # - RETRY_FETCHES +# - Branch-specific stuff # - handle local manifests # - Sync mirror if we're using one -# - Branch-specific stuff # - main sync and build loop # For each device in `$DEVICE_LIST` # - setup subdirectories @@ -134,6 +134,27 @@ if [ -n "${RETRY_FETCHES-}" ]; then fi fi +# Branch-specific stuff +branch_dir=${branch//[^[:alnum:]]/_} +branch_dir=${branch_dir^^} + +if [ -n "$branch" ] && [ -n "$devices" ]; then + case "$branch" in + lineage-21.0*) + themuppets_branch="lineage-21.0" + android_version="14" + ;; + *) + echo ">> [$(date)] Building branch $branch is not (yet) suppported" + exit 1 + ;; + esac + android_version_major=$(cut -d '.' -f 1 <<< $android_version) + + mkdir -p "$SRC_DIR/$branch_dir" + cd "$SRC_DIR/$branch_dir" +fi + # Handle local manifests ## Copy local manifests echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'" @@ -168,26 +189,6 @@ if [ "$LOCAL_MIRROR" = true ]; then fi fi -# Branch-specific stuff -branch_dir=${branch//[^[:alnum:]]/_} -branch_dir=${branch_dir^^} - -if [ -n "$branch" ] && [ -n "$devices" ]; then - case "$branch" in - lineage-21.0*) - themuppets_branch="lineage-21.0" - android_version="14" - ;; - *) - echo ">> [$(date)] Building branch $branch is not (yet) suppported" - exit 1 - ;; - esac - android_version_major=$(cut -d '.' -f 1 <<< $android_version) - - mkdir -p "$SRC_DIR/$branch_dir" - cd "$SRC_DIR/$branch_dir" -fi # - main sync and build loop # For each device in `$DEVICE_LIST` From 38000b66289ca0562386f615c70d084b23d10474 Mon Sep 17 00:00:00 2001 From: Pete Date: Mon, 2 Sep 2024 18:01:53 +0100 Subject: [PATCH 30/43] Set DEBUG_LOG after setup overlays --- src/new_build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index ed83d08..69792d3 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -209,7 +209,6 @@ for codename in ${devices//,/ }; do else logsubdir= fi - DEBUG_LOG="$LOGS_DIR/$logsubdir/lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename.log" # `repo init` # ToDo: do we need to add REPO_VERSION - see https://github.com/lineageos-infra/build-config/commit/312e3242d04db35945ce815ab35864a86b14b866 @@ -268,6 +267,8 @@ for codename in ${devices//,/ }; do los_ver_minor=$(sed -n -e 's/^\s*PRODUCT_VERSION_MINOR = //p' "$makefile_containing_version") los_ver="$los_ver_major.$los_ver_minor" + DEBUG_LOG="$LOGS_DIR/$logsubdir/lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename.log" + # Add custom packages to be installed if [ -n "$CUSTOM_PACKAGES" ]; then echo ">> [$(date)] Adding custom packages ($CUSTOM_PACKAGES)" From c1e0512c9f57c65db5f8aa5b993f5d98e34575e2 Mon Sep 17 00:00:00 2001 From: Pete Date: Mon, 2 Sep 2024 18:10:27 +0100 Subject: [PATCH 31/43] cd to SRCDIR/branchdir before handling manifests" --- src/new_build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index 69792d3..192b331 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -138,6 +138,9 @@ fi branch_dir=${branch//[^[:alnum:]]/_} branch_dir=${branch_dir^^} +mkdir -p "$SRC_DIR/$branch_dir" +cd "$SRC_DIR/$branch_dir" + if [ -n "$branch" ] && [ -n "$devices" ]; then case "$branch" in lineage-21.0*) @@ -150,9 +153,6 @@ if [ -n "$branch" ] && [ -n "$devices" ]; then ;; esac android_version_major=$(cut -d '.' -f 1 <<< $android_version) - - mkdir -p "$SRC_DIR/$branch_dir" - cd "$SRC_DIR/$branch_dir" fi # Handle local manifests From eb2e8ae2ec0356826cc79b50f3df6105b1d9a85f Mon Sep 17 00:00:00 2001 From: Pete Date: Mon, 2 Sep 2024 18:04:54 +0100 Subject: [PATCH 32/43] Initialise userscriptfail --- src/new_build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index 192b331..bebe9ce 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -299,7 +299,9 @@ for codename in ${devices//,/ }; do else echo ">> [$(date)] Preparing build environment disabled" fi - + + userscriptfail=false + # Call `before.sh` if [ -f /root/userscripts/before.sh ]; then echo ">> [$(date)] Running before.sh" From 707508ddbe37654c306aabacdfa737e810f885e2 Mon Sep 17 00:00:00 2001 From: Pete Date: Mon, 2 Sep 2024 18:32:22 +0100 Subject: [PATCH 33/43] DEVICE -> codename --- src/new_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index bebe9ce..713e008 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -215,9 +215,9 @@ for codename in ${devices//,/ }; do if [ "$CALL_REPO_INIT" = true ]; then echo ">> [$(date)] (Re)initializing branch repository" | tee -a "$repo_log" if [ "$LOCAL_MIRROR" = true ]; then - ( yes||: ) | repo init -u https://github.com/LineageOS/android.git --reference "$MIRROR_DIR" -b "$branch" -g default,-darwin,-muppets,muppets_"${DEVICE}" --git-lfs &>> "$repo_log" + ( yes||: ) | repo init -u https://github.com/LineageOS/android.git --reference "$MIRROR_DIR" -b "$branch" -g default,-darwin,-muppets,muppets_"${codename}" --git-lfs &>> "$repo_log" else - ( yes||: ) | repo init -u https://github.com/LineageOS/android.git -b "$branch" -g default,-darwin,-muppets,muppets_"${DEVICE}" --git-lfs &>> "$repo_log" + ( yes||: ) | repo init -u https://github.com/LineageOS/android.git -b "$branch" -g default,-darwin,-muppets,muppets_"${codename}" --git-lfs &>> "$repo_log" fi else echo ">> [$(date)] Calling repo init disabled" From b8705d2b99c4e34ffdc33d483ca77fc8d403a193 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 2 Sep 2024 21:57:15 +0100 Subject: [PATCH 34/43] Tweak calls to repo sync - add retry-fetches - -c to --current-branch --- src/new_build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index 713e008..cb365dc 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -183,7 +183,7 @@ if [ "$LOCAL_MIRROR" = true ]; then fi if [ "$SYNC_MIRROR" = true ]; then echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log" - repo sync "${jobs_arg[@]}" --force-sync --no-clone-bundle &>> "$repo_log" + repo sync "${jobs_arg[@]}" "${retry_fetches_arg[@]}" --force-sync --no-clone-bundle &>> "$repo_log" else echo ">> [$(date)] Sync mirror repository disabled" | tee -a "$repo_log" fi @@ -226,7 +226,7 @@ for codename in ${devices//,/ }; do # `repo sync` if [ "$CALL_REPO_SYNC" = true ]; then echo ">> [$(date)] Syncing branch repository" | tee -a "$repo_log" - repo sync "${jobs_arg[@]}" -c --force-sync &>> "$repo_log" + repo sync "${jobs_arg[@]}" "${retry_fetches_arg[@]}" --current-branch --force-sync &>> "$repo_log" else echo ">> [$(date)] Syncing branch repository disabled" | tee -a "$repo_log" fi @@ -299,9 +299,9 @@ for codename in ${devices//,/ }; do else echo ">> [$(date)] Preparing build environment disabled" fi - + userscriptfail=false - + # Call `before.sh` if [ -f /root/userscripts/before.sh ]; then echo ">> [$(date)] Running before.sh" From 498db885dec317a8286303417afa56c326f5273b Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Thu, 5 Sep 2024 16:01:32 +0100 Subject: [PATCH 35/43] We dont need to use REPO_VERSION --- src/new_build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/new_build.sh b/src/new_build.sh index cb365dc..2ec9256 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -211,7 +211,6 @@ for codename in ${devices//,/ }; do fi # `repo init` - # ToDo: do we need to add REPO_VERSION - see https://github.com/lineageos-infra/build-config/commit/312e3242d04db35945ce815ab35864a86b14b866 if [ "$CALL_REPO_INIT" = true ]; then echo ">> [$(date)] (Re)initializing branch repository" | tee -a "$repo_log" if [ "$LOCAL_MIRROR" = true ]; then From 566dcf4d566d5b7ee7428112fa703c495f32356b Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Thu, 5 Sep 2024 16:15:32 +0100 Subject: [PATCH 36/43] Disable LOCAL_MIROR & BUILD_OVERLAY for now --- src/new_build.sh | 48 ++++++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/src/new_build.sh b/src/new_build.sh index 2ec9256..23df685 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -171,22 +171,24 @@ fi # Sync mirror if we're using one if [ "$LOCAL_MIRROR" = true ]; then + echo "Using LOCAL_MIRROR is not yet working" - cd "$MIRROR_DIR" - if [ "$INIT_MIRROR" = true ]; then - if [ ! -d .repo ]; then - echo ">> [$(date)] Initializing mirror repository" | tee -a "$repo_log" - ( yes||: ) | repo init -u https://github.com/LineageOS/mirror --mirror --no-clone-bundle -p linux --git-lfs &>> "$repo_log" - fi - else - echo ">> [$(date)] Initializing mirror repository disabled" | tee -a "$repo_log" - fi - if [ "$SYNC_MIRROR" = true ]; then - echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log" - repo sync "${jobs_arg[@]}" "${retry_fetches_arg[@]}" --force-sync --no-clone-bundle &>> "$repo_log" - else - echo ">> [$(date)] Sync mirror repository disabled" | tee -a "$repo_log" - fi + # cd "$MIRROR_DIR" + # if [ "$INIT_MIRROR" = true ]; then + # if [ ! -d .repo ]; then + # echo ">> [$(date)] Initializing mirror repository" | tee -a "$repo_log" + # ( yes||: ) | repo init -u https://github.com/LineageOS/mirror --mirror --no-clone-bundle -p linux --git-lfs &>> "$repo_log" + # fi + # else + # echo ">> [$(date)] Initializing mirror repository disabled" | tee -a "$repo_log" + # fi + # if [ "$SYNC_MIRROR" = true ]; then + # echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log" + # repo sync "${jobs_arg[@]}" "${retry_fetches_arg[@]}" --force-sync --no-clone-bundle &>> "$repo_log" + # + # else + # echo ">> [$(date)] Sync mirror repository disabled" | tee -a "$repo_log" + # fi fi @@ -244,13 +246,15 @@ for codename in ${devices//,/ }; do # Setup our overlays if [ "$BUILD_OVERLAY" = true ]; then - lowerdir=$SRC_DIR/$branch_dir - upperdir=$TMP_DIR/device - workdir=$TMP_DIR/workdir - merged=$TMP_DIR/merged - mkdir -p "$upperdir" "$workdir" "$merged" - mount -t overlay overlay -o lowerdir="$lowerdir",upperdir="$upperdir",workdir="$workdir" "$merged" - source_dir="$TMP_DIR/merged" + echo "Using BUILD_OVERLAY is not yet working" + + # lowerdir=$SRC_DIR/$branch_dir + # upperdir=$TMP_DIR/device + # workdir=$TMP_DIR/workdir + # merged=$TMP_DIR/merged + # mkdir -p "$upperdir" "$workdir" "$merged" + # mount -t overlay overlay -o lowerdir="$lowerdir",upperdir="$upperdir",workdir="$workdir" "$merged" + # source_dir="$TMP_DIR/merged" else source_dir="$SRC_DIR/$branch_dir" fi From 2475c7d9973c46f7069ce9a85f7c68751a433c25 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Thu, 5 Sep 2024 18:23:33 +0100 Subject: [PATCH 37/43] Set RELEASE_TYPE --- src/new_build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index 23df685..ba9e357 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -34,6 +34,7 @@ # - `repo init` # - `repo sync` # - setup our overlays +# - Set RELEASE_TYPE # - Add custom packages to be installed # - Handle keys # - Prepare the environment @@ -272,6 +273,11 @@ for codename in ${devices//,/ }; do DEBUG_LOG="$LOGS_DIR/$logsubdir/lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename.log" + # Set RELEASE_TYPE + echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type" + sed -i "/\$(filter .*\$(${vendor^^}_BUILDTYPE)/,/endif/d" "$makefile_containing_version" + + # Add custom packages to be installed if [ -n "$CUSTOM_PACKAGES" ]; then echo ">> [$(date)] Adding custom packages ($CUSTOM_PACKAGES)" From a311f3967f77c01f034ce349699a2c0cdaf1f951 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Thu, 5 Sep 2024 18:26:43 +0100 Subject: [PATCH 38/43] Handle OTA_URL --- src/new_build.sh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index ba9e357..ec3b05d 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -35,6 +35,7 @@ # - `repo sync` # - setup our overlays # - Set RELEASE_TYPE +# - Handle OTA_URL # - Add custom packages to be installed # - Handle keys # - Prepare the environment @@ -277,6 +278,34 @@ for codename in ${devices//,/ }; do echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type" sed -i "/\$(filter .*\$(${vendor^^}_BUILDTYPE)/,/endif/d" "$makefile_containing_version" + # Set a custom updater URI if a OTA URL is provided + echo ">> [$(date)] Adding OTA URL overlay (for custom URL $OTA_URL)" + if [ -n "$OTA_URL" ]; then + if [ -d "packages/apps/Updater/app/src/main/res/values" ]; then + # "New" Updater project structure + updater_values_dir="packages/apps/Updater/app/src/main/res/values" + elif [ -d "packages/apps/Updater/res/values" ]; then + # "Old" Updater project structure + updater_values_dir="packages/apps/Updater/res/values" + else + echo ">> [$(date)] ERROR: no 'values' dir of Updater app found" + exit 1 + fi + + updater_url_overlay_dir="vendor/$vendor/overlay/microg/${updater_values_dir}/" + mkdir -p "$updater_url_overlay_dir" + + if grep -q updater_server_url ${updater_values_dir}/strings.xml; then + # "New" updater configuration: full URL (with placeholders {device}, {type} and {incr}) + sed "s|{name}|updater_server_url|g; s|{url}|$OTA_URL/v1/{device}/{type}/{incr}|g" /root/packages_updater_strings.xml > "$updater_url_overlay_dir/strings.xml" + elif grep -q conf_update_server_url_def ${updater_values_dir}/strings.xml; then + # "Old" updater configuration: just the URL + sed "s|{name}|conf_update_server_url_def|g; s|{url}|$OTA_URL|g" /root/packages_updater_strings.xml > "$updater_url_overlay_dir/strings.xml" + else + echo ">> [$(date)] ERROR: no known Updater URL property found" + exit 1 + fi + fi # Add custom packages to be installed if [ -n "$CUSTOM_PACKAGES" ]; then From 68490248cc8af518813c9bd755a5bfead7f41b67 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Sun, 15 Sep 2024 12:55:56 +0100 Subject: [PATCH 39/43] Use new_build.sh for 20.0 builds --- src/init.sh | 12 ++++++++---- src/new_build.sh | 4 ++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/init.sh b/src/init.sh index bcc63ae..04b7625 100755 --- a/src/init.sh +++ b/src/init.sh @@ -65,10 +65,14 @@ visibility = ["//visibility:public"], ) _EOB -build_file="build.sh" -if [ "$BRANCH_NAME" = "lineage-21.0" ]; then - build_file="new_build.sh" -fi +case "$BRANCH_NAME" in + "lineage-20.0" | "lineage-21.0" ) + build_file="new_build.sh" + ;; + * ) + build_file="build.sh" + ;; +esac if [ "$CRONTAB_TIME" = "now" ]; then /root/$build_file diff --git a/src/new_build.sh b/src/new_build.sh index ec3b05d..f9dd912 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -145,6 +145,10 @@ cd "$SRC_DIR/$branch_dir" if [ -n "$branch" ] && [ -n "$devices" ]; then case "$branch" in + lineage-20.0*) + themuppets_branch="lineage-20.0" + android_version="13" + ;; lineage-21.0*) themuppets_branch="lineage-21.0" android_version="14" From e697d50d350811b7f779141217f58f72636b19b1 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Sun, 15 Sep 2024 19:49:54 +0100 Subject: [PATCH 40/43] Exit if BUILD_OVERLAY set true --- src/new_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/new_build.sh b/src/new_build.sh index f9dd912..6b6baf3 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -253,6 +253,7 @@ for codename in ${devices//,/ }; do # Setup our overlays if [ "$BUILD_OVERLAY" = true ]; then echo "Using BUILD_OVERLAY is not yet working" + exit 1 # lowerdir=$SRC_DIR/$branch_dir # upperdir=$TMP_DIR/device From e4b6156ab310fc4b17b7e7417dbc84c62f3ad9dd Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Sun, 15 Sep 2024 23:01:03 +0100 Subject: [PATCH 41/43] Check for non-working environment variable values --- src/new_build.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/new_build.sh b/src/new_build.sh index 6b6baf3..0b700e3 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -108,6 +108,17 @@ echo ">> [$(date)] Devices: $devices" vendor=lineage +## Check for non-working environment variable values +if [ "$LOCAL_MIRROR" = true ]; then + echo "Using LOCAL_MIRROR is not yet working" + exit 1 +fi + +if [ "$BUILD_OVERLAY" = true ]; then + echo "Using BUILD_OVERLAY is not yet working" + exit 1 +fi + ## CLEAN_OUTDIR if [ "$CLEAN_OUTDIR" = true ]; then echo ">> [$(date)] Cleaning '$ZIP_DIR'" From a295c7f2e0cd10d6cf362c319af102f0c537ee57 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Sat, 21 Sep 2024 15:33:20 +0100 Subject: [PATCH 42/43] Rename build.sh -> legacy-build.sh --- src/init.sh | 2 +- src/{build.sh => legacy-build.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{build.sh => legacy-build.sh} (100%) diff --git a/src/init.sh b/src/init.sh index 04b7625..0ef354a 100755 --- a/src/init.sh +++ b/src/init.sh @@ -70,7 +70,7 @@ case "$BRANCH_NAME" in build_file="new_build.sh" ;; * ) - build_file="build.sh" + build_file="legacy-build.sh" ;; esac diff --git a/src/build.sh b/src/legacy-build.sh similarity index 100% rename from src/build.sh rename to src/legacy-build.sh From 8addf1b015296324b718b430dbb15c7599f996dd Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Sat, 21 Sep 2024 15:41:27 +0100 Subject: [PATCH 43/43] Use restructured build for 19.1 Complete fix for ##682 --- src/init.sh | 2 +- src/new_build.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/init.sh b/src/init.sh index 0ef354a..3593cdd 100755 --- a/src/init.sh +++ b/src/init.sh @@ -66,7 +66,7 @@ visibility = ["//visibility:public"], _EOB case "$BRANCH_NAME" in - "lineage-20.0" | "lineage-21.0" ) + "lineage-19.1" | "lineage-20.0" | "lineage-21.0" ) build_file="new_build.sh" ;; * ) diff --git a/src/new_build.sh b/src/new_build.sh index 0b700e3..f06e95e 100755 --- a/src/new_build.sh +++ b/src/new_build.sh @@ -156,6 +156,10 @@ cd "$SRC_DIR/$branch_dir" if [ -n "$branch" ] && [ -n "$devices" ]; then case "$branch" in + lineage-19.1*) + themuppets_branch="lineage-19.1" + android_version="12" + ;; lineage-20.0*) themuppets_branch="lineage-20.0" android_version="13"