Merge branch 'experimental' into lineage-21

Dockerfile

@@ -121,6 +121,21 @@ ENV REPO_INIT_ARGS ""
 # Allowed values: positive, non-null integers
 ENV RETRY_FETCHES=
 
+
+# variables to control whether or not tasks are implemented
+ENV INIT_MIRROR true
+ENV SYNC_MIRROR true
+ENV RESET_VENDOR_UNDO_PATCHES true
+ENV CALL_REPO_INIT true
+ENV CALL_REPO_SYNC true
+ENV CALL_GIT_LFS_PULL false
+ENV APPLY_PATCHES true
+ENV PREPARE_BUILD_ENVIRONMENT true
+ENV CALL_BREAKFAST true
+ENV CALL_MKA true
+ENV ZIP_UP_IMAGES false
+ENV MAKE_IMG_ZIP_FILE false
+
 # You can optionally specify a USERSCRIPTS_DIR volume containing these scripts:
 #  * begin.sh, run at the very beginning
 #  * before.sh, run after the syncing and patching, before starting the builds

README.md

@@ -72,7 +72,7 @@ to the LineageOS official builds, just signed with the test keys.
 When multiple branches are selected, use `DEVICE_LIST_<BRANCH_NAME>` to specify
 the list of devices for each specific branch (see [the examples](#examples)).
 
-### GMS / microG
+#### GMS / microG
 
 To include microG (or possibly the actual Google Mobile Services) in your build,
 LineageOS expects certain Makefiles in `vendor/partner_gms` and variable
@@ -90,7 +90,7 @@ official lineageos4microg builds. To include it in your build, create an XML
 </manifest>
 ```
 
-### Additional custom apps
+#### Additional custom apps
 
 If you wish to add other apps to your ROM, you can include a repository with
 source code or prebuilt APKs. For prebuilt apks, see the [android_vendor_partner_gms][android_vendor_partner_gms]
@@ -108,7 +108,7 @@ Include the repo with another manifest file like this:
 And when starting the build, set the `CUSTOM_PACKAGES` variable to a list of app names
 (defined by `LOCAL_MODULE` in `Android.mk`) separated by spaces.
 
-### Signature spoofing
+#### Signature spoofing
 
 There are two options for the [signature spoofing patch][signature-spoofing]
 required for [microG][microg]:
@@ -134,7 +134,7 @@ FAKE_SIGNATURE permission must be included in the build as system apps
 (e.g. as part of GMS or `CUSTOM_PACKAGES`)
 
 
-### Proprietary files
+#### Proprietary files
 
 Some proprietary files are needed to create a LineageOS build, but they're not
 included in the LineageOS repo for legal reasons. You can obtain these blobs in
@@ -149,7 +149,7 @@ The third way is the easiest one and is enabled by default; if you're OK with
 that just move on, otherwise set `INCLUDE_PROPRIETARY (true)` to `false` and
 manually provide the blobs (not explained in this guide).
 
-### Over the Air updates
+#### Over the Air updates
 
 To enable OTA for you builds, you need to run a server that speaks the protocol
 understood by the [LineageOS updater app][updater] and provide the URL to this
@@ -169,7 +169,7 @@ image. Follow these steps to prepare your builds for OTA:
 If you don't setup a OTA server you won't be able to update the device from the
 updater app (but you can still update it manually with the recovery of course).
 
-### Signing
+#### Signing
 
 By default, builds are signed with the Android test keys. If you want to sign
 your builds with your own keys (**highly recommended**):
@@ -177,15 +177,40 @@ your builds with your own keys (**highly recommended**):
  * `SIGN_BUILDS (false)`: set to `true` to sign the builds with the keys
     contained in `/srv/keys`; if no keys are present, a new set will be generated
 
-### Other settings
+#### Settings to control 'switchable' build steps
+
+Some of the the steps in the build process (e.g `repo sync`, `mka`) can take a long time to complete. When working on a build, it may be desirable to skip some of the steps. The following environment variables (and their default values) control whether or not each step is performed
+```
+# variables to control whether or not tasks are implemented
+ENV INIT_MIRROR true
+ENV SYNC_MIRROR true
+ENV RESET_VENDOR_UNDO_PATCHES true
+ENV CALL_REPO_INIT true
+ENV CALL_REPO_SYNC true
+ENV CALL_GIT_LFS_PULL false
+ENV APPLY_PATCHES true
+ENV PREPARE_BUILD_ENVIRONMENT true
+ENV CALL_BREAKFAST true
+ENV CALL_MKA true
+ENV ZIP_UP_IMAGES false
+ENV MAKE_IMG_ZIP_FILE false
+```
+
+To `switch` an operation, change the default value of the the variable in a `-e clause` in the `docker run` command e.g.
+` -e "CALL_REPO-SYNC=false" \`
+
+The `ZIP_UP_IMAGES` and `MAKE_IMG_ZIP_FILE` variables control how the `.img` files created by the buid are handled:
+- by default, the `img` files are copied - unzipped - to the `zips` directory
+- if `ZIP_UP_IMAGES` is set `true`, the images are zipped and the resulting `...images.zip` is copied to the `zips` directory
+- if `MAKE_IMG_ZIP_FILE` is set `true`, a flashsable `...-img.zip` file is created, which can be installed using `fastboot flash` or `fastboot update`
+
+
+#### Other settings
 
 Other useful settings are:
 
  * `CCACHE_SIZE (50G)`: change this if you want to give more (or less) space to
     ccache
- * `WITH_SU (false)`: set to `true` to embed `su` in the build (note that, even
-    when set to `false`, you can still enable root by flashing the
-    [su installable ZIP][los-extras]). This is only for lineage version 16 and below.
  * `RELEASE_TYPE (UNOFFICIAL)`: change the release type of your builds
  * `BUILD_TYPE (userdebug)`: type of your builds, see [Android docs](https://source.android.com/docs/setup/build/building#choose-a-target)
  * `BUILD_OVERLAY (false)`: normally each build is done on the source tree, then

src/build.sh

@@ -76,15 +76,17 @@ if [ -n "${RETRY_FETCHES-}" ]; then
   fi
 fi
 
-
 if [ "$LOCAL_MIRROR" = true ]; then
 
   cd "$MIRROR_DIR"
-
+  if [ "$INIT_MIRROR" = true ]; then
     if [ ! -d .repo ]; then
       echo ">> [$(date)] Initializing mirror repository" | tee -a "$repo_log"
       ( yes||: ) | repo init -u https://github.com/LineageOS/mirror --mirror --no-clone-bundle -p linux --git-lfs &>> "$repo_log"
     fi
+  else
+    echo ">> [$(date)] Initializing mirror repository disabled" | tee -a "$repo_log"
+  fi
 
   # Copy local manifests to the appropriate folder in order take them into consideration
   echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'"
@@ -98,8 +100,12 @@ if [ "$LOCAL_MIRROR" = true ]; then
       "https://gitlab.com/the-muppets/manifest/raw/mirror/default.xml" .repo/local_manifests/proprietary_gitlab.xml
   fi
 
+  if [ "$SYNC_MIRROR" = true ]; then
     echo ">> [$(date)] Syncing mirror repository" | tee -a "$repo_log"
-  repo sync "${jobs_arg[@]}" "${retry_fetches_arg[@]}" --force-sync --no-clone-bundle &>> "$repo_log"
+    repo sync "${jobs_arg[@]}" --force-sync --no-clone-bundle &>> "$repo_log"
+  else
+    echo ">> [$(date)] Sync mirror repository disabled" | tee -a "$repo_log"
+  fi
 fi
 
 for branch in ${BRANCH_NAME//,/ }; do
@@ -172,6 +178,7 @@ for branch in ${BRANCH_NAME//,/ }; do
     echo ">> [$(date)] Branch:  $branch"
     echo ">> [$(date)] Devices: $devices"
 
+    if [ "$RESET_VENDOR_UNDO_PATCHES" = true ]; then
       # Remove previous changes of vendor/cm, vendor/lineage and frameworks/base (if they exist)
       # TODO: maybe reset everything using https://source.android.com/setup/develop/repo#forall
       for path in "vendor/cm" "vendor/lineage" "frameworks/base" "packages/apps/PermissionController" "packages/modules/Permission"; do
@@ -182,13 +189,20 @@ for branch in ${BRANCH_NAME//,/ }; do
           cd "$SRC_DIR/$branch_dir"
         fi
       done
+    else
+      echo ">> [$(date)] Resetting vendor and undoing patches disabled" | tee -a "$repo_log"
+    fi
 
+    if [ "$CALL_REPO_INIT" = true ]; then
       echo ">> [$(date)] (Re)initializing branch repository" | tee -a "$repo_log"
       if [ "$LOCAL_MIRROR" = true ]; then
         ( yes||: ) | repo init -u https://github.com/LineageOS/android.git --reference "$MIRROR_DIR" -b "$branch" --git-lfs &>> "$repo_log"
       else
         ( yes||: ) | repo init -u https://github.com/LineageOS/android.git -b "$branch" --git-lfs &>> "$repo_log"
       fi
+    else
+      echo ">> [$(date)] Calling repo init disabled"
+    fi
 
     # Copy local manifests to the appropriate folder in order take them into consideration
     echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '.repo/local_manifests/'"
@@ -202,9 +216,20 @@ for branch in ${BRANCH_NAME//,/ }; do
         "https://gitlab.com/the-muppets/manifest/raw/$themuppets_branch/muppets.xml" .repo/local_manifests/proprietary_gitlab.xml
     fi
 
-    echo ">> [$(date)] Syncing branch repository" | tee -a "$repo_log"
     builddate=$(date +%Y%m%d)
-    repo sync "${jobs_arg[@]}" "${retry_fetches_arg[@]}" -c --force-sync &>> "$repo_log"
+    if [ "$CALL_REPO_SYNC" = true ]; then
+      echo ">> [$(date)] Syncing branch repository" | tee -a "$repo_log"
+      repo sync "${jobs_arg[@]}" -c --force-sync &>> "$repo_log"
+    else
+      echo ">> [$(date)] Syncing branch repository disabled" | tee -a "$repo_log"
+    fi
+
+    if [ "$CALL_GIT_LFS_PULL" = true ]; then
+      echo ">> [$(date)] Calling git lfs pull" | tee -a "$repo_log"
+      repo forall -v -c git lfs pull &>> "$repo_log"
+    else
+      echo ">> [$(date)] Calling git lfs pull disabled" | tee -a "$repo_log"
+    fi
 
     if [ ! -d "vendor/$vendor" ]; then
       echo ">> [$(date)] Missing \"vendor/$vendor\", aborting"
@@ -223,6 +248,7 @@ for branch in ${BRANCH_NAME//,/ }; do
     los_ver_minor=$(sed -n -e 's/^\s*PRODUCT_VERSION_MINOR = //p' "$makefile_containing_version")
     los_ver="$los_ver_major.$los_ver_minor"
 
+    if [ "$APPLY_PATCHES" = true ]; then
     # If needed, apply the microG's signature spoofing patch
       if [ "$SIGNATURE_SPOOFING" = "yes" ] || [ "$SIGNATURE_SPOOFING" = "restricted" ]; then
         # Determine which patch should be applied to the current Android source tree
@@ -258,6 +284,9 @@ for branch in ${BRANCH_NAME//,/ }; do
         mkdir -p "vendor/$vendor/overlay/microg/frameworks/base/core/res/res/values/"
         cp /root/signature_spoofing_patches/frameworks_base_config.xml "vendor/$vendor/overlay/microg/frameworks/base/core/res/res/values/config.xml"
       fi
+    else
+      echo ">> [$(date)] Applying patches disabled"
+    fi
 
     echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type"
     sed -i "/\$(filter .*\$(${vendor^^}_BUILDTYPE)/,/endif/d" "$makefile_containing_version"
@@ -310,12 +339,16 @@ for branch in ${BRANCH_NAME//,/ }; do
       fi
     fi
 
+    if [ "$PREPARE_BUILD_ENVIRONMENT" = true ]; then
       # Prepare the environment
       echo ">> [$(date)] Preparing build environment"
       set +eu
       # shellcheck source=/dev/null
       source build/envsetup.sh > /dev/null
       set -eu
+    else
+      echo ">> [$(date)] Preparing build environment disabled"
+    fi
 
     if [ -f /root/userscripts/before.sh ]; then
       echo ">> [$(date)] Running before.sh"
@@ -355,10 +388,16 @@ for branch in ${BRANCH_NAME//,/ }; do
 
         DEBUG_LOG="$LOGS_DIR/$logsubdir/lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename.log"
 
+        breakfast_returncode=0
+        if [ "$CALL_BREAKFAST" = true ]; then
           set +eu
           breakfast "$codename" "$BUILD_TYPE" &>> "$DEBUG_LOG"
           breakfast_returncode=$?
           set -eu
+        else
+          echo ">> [$(date)] Calling breakfast disabled"
+        fi
+
         if [ $breakfast_returncode -ne 0 ]; then
             echo ">> [$(date)] breakfast failed for $codename, $branch branch" | tee -a "$DEBUG_LOG"
             # call post-build.sh so the failure is logged in a way that is more visible
@@ -374,22 +413,60 @@ for branch in ${BRANCH_NAME//,/ }; do
           /root/userscripts/pre-build.sh "$codename" &>> "$DEBUG_LOG" || echo ">> [$(date)] Warning: pre-build.sh failed!"
         fi
 
+        build_successful=true
+        if [ "$CALL_MKA" = true ]; then
           # Start the build
           echo ">> [$(date)] Starting build for $codename, $branch branch" | tee -a "$DEBUG_LOG"
           build_successful=false
-        if (set +eu ; mka "${jobs_arg[@]}" bacon) &>> "$DEBUG_LOG"; then
+          files_to_hash=()
 
-          # Move produced ZIP files to the main OUT directory
+          if (set +eu ; mka "${jobs_arg[@]}" bacon) &>> "$DEBUG_LOG"; then
+            if [ "$MAKE_IMG_ZIP_FILE" = true ]; then
+              # make the `-img.zip` file
+              echo ">> [$(date)] Making -img.zip file" | tee -a "$DEBUG_LOG"
+              infile="out/target/product/$codename/obj/PACKAGING/target_files_intermediates/lineage_$codename-target_files-eng.root.zip"
+              img_zip_file="lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename-img.zip"
+              img_from_target_files "$infile" "$img_zip_file"  &>> "$DEBUG_LOG"
+
+              # move it to the zips directory
+              mv "$img_zip_file" "$ZIP_DIR/$zipsubdir/" &>> "$DEBUG_LOG"
+              files_to_hash+=( "$img_zip_file" )
+            else
+              echo ">> [$(date)] Making -img.zip file disabled"
+            fi
+
+          # Move the ROM zip files to the main OUT directory
           echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a "$DEBUG_LOG"
           cd out/target/product/"$codename"
-          files_to_hash=()
+
           for build in lineage-*.zip; do
             cp -v system/build.prop "$ZIP_DIR/$zipsubdir/$build.prop" &>> "$DEBUG_LOG"
             mv "$build" "$ZIP_DIR/$zipsubdir/" &>> "$DEBUG_LOG"
             files_to_hash+=( "$build" )
           done
+
+          cd "$source_dir/out/target/product/$codename/obj/PACKAGING/target_files_intermediates/lineage_$codename-target_files-eng.root/IMAGES/"
+          if [ "$ZIP_UP_IMAGES" = true ]; then
+            # zipping the .img files
+            echo ">> [$(date)] Zipping the .img files" | tee -a "$DEBUG_LOG"
+
+            files_to_zip=()
+            images_zip_file="lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename-images.zip"
             cd "$source_dir/out/target/product/$codename/obj/PACKAGING/target_files_intermediates/lineage_$codename-target_files-eng.root/IMAGES/"
 
+            for image in recovery boot vendor_boot dtbo super_empty vbmeta vendor_kernel_boot; do
+              if [ -f "$image.img" ]; then
+                echo ">> [$(date)] Adding $image.img" to "$images_zip_file" | tee -a "$DEBUG_LOG"
+                files_to_zip+=( "$image.img" )
+              fi
+            done
+
+            zip "$images_zip_file" "${files_to_zip[@]}"
+            mv "$images_zip_file" "$ZIP_DIR/$zipsubdir/"
+            files_to_hash+=( "$images_zip_file" )
+          else
+            # just copy the mages to the zips directory
+            echo ">> [$(date)] Zipping the '-img' files disabled"
             for image in recovery boot vendor_boot dtbo super_empty vbmeta vendor_kernel_boot; do
               if [ -f "$image.img" ]; then
                 recovery_name="lineage-$los_ver-$builddate-$RELEASE_TYPE-$codename-$image.img"
@@ -398,6 +475,8 @@ for branch in ${BRANCH_NAME//,/ }; do
                 files_to_hash+=( "$recovery_name" )
               fi
             done
+          fi
+
           cd "$ZIP_DIR/$zipsubdir"
           for f in "${files_to_hash[@]}"; do
             sha256sum "$f" > "$ZIP_DIR/$zipsubdir/$f.sha256sum"
@@ -407,6 +486,10 @@ for branch in ${BRANCH_NAME//,/ }; do
           else
             echo ">> [$(date)] Failed build for $codename" | tee -a "$DEBUG_LOG"
           fi
+        else
+          echo ">> [$(date)] Calling mka for $codename, $branch branch disabled"
+        fi
+      fi
 
         # Remove old zips and logs
         if [ "$DELETE_OLD_ZIPS" -gt "0" ]; then
@@ -425,7 +508,7 @@ for branch in ${BRANCH_NAME//,/ }; do
         fi
         if [ -f /root/userscripts/post-build.sh ]; then
           echo ">> [$(date)] Running post-build.sh for $codename" >> "$DEBUG_LOG"
-          /root/userscripts/post-build.sh "$codename" $build_successful "$branch" &>> "$DEBUG_LOG" || echo ">> [$(date)] Warning: post-build.sh failed!"
+          /root/userscripts/post-build.sh "$codename" "$build_successful" "$branch" &>> "$DEBUG_LOG" || echo ">> [$(date)] Warning: post-build.sh failed!"
         fi
         echo ">> [$(date)] Finishing build for $codename" | tee -a "$DEBUG_LOG"
 
@@ -454,10 +537,7 @@ for branch in ${BRANCH_NAME//,/ }; do
             (set +eu ; mka "${jobs_arg[@]}" clean) &>> "$DEBUG_LOG"
           fi
         fi
-
-      fi
     done
-
   fi
 done