Improve the logging

2024-09-20 12:22:40 +02:00 · 2017-10-18 17:20:32 +02:00 · 2017-10-18 17:20:32 +02:00 · d912e4b030
commit d912e4b030
parent 53250fcbe3
4 changed files with 74 additions and 65 deletions
--- a/12
+++ b/12
@ -10,6 +10,7 @@ ENV ZIP_DIR /srv/zips
 ENV LMANIFEST_DIR /srv/local_manifests
 ENV DELTA_DIR /srv/delta
 ENV KEYS_DIR /srv/keys
+ENV LOGS_DIR /srv/logs

 ENV DEBIAN_FRONTEND noninteractive
 ENV USER root
@ -51,9 +52,6 @@ ENV CLEAN_OUTDIR false
 # For example, '0 10 * * *' means 'Every day at 10:00 UTC'
 ENV CRONTAB_TIME 'now'

-# Print detailed output rather than only summary
-ENV DEBUG false
-
 # Clean artifacts output after each build
 ENV CLEAN_AFTER_BUILD true

@ -72,6 +70,9 @@ ENV SIGN_BUILDS false
 # Move the resulting zips to $ZIP_DIR/$codename instead of $ZIP_DIR/
 ENV ZIP_SUBDIR true

+# Write the verbose logs to $LOGS_DIR/$codename instead of $LOGS_DIR/
+ENV LOGS_SUBDIR true
+
 # Apply the MicroG's signature spoofing patch
 # Valid values are "no", "yes" (for the original MicroG's patch) and
 # "restricted" (to grant the permission only to the system privileged apps).
@ -92,6 +93,9 @@ ENV DELETE_OLD_ZIPS 0
 # Delete old deltas in $DELTA_DIR, keep only the N latest one (0 to disable)
 ENV DELETE_OLD_DELTAS 0

+# Delete old logs in $LOGS_DIR, keep only the N latest one (0 to disable)
+ENV DELETE_OLD_LOGS 0
+
 # Create a JSON file that indexes the build zips at the end of the build process
 # (for the updates in OpenDelta). The file will be created in $ZIP_DIR with the
 # specified name; leave empty to skip it.
@ -106,6 +110,7 @@ VOLUME $ZIP_DIR
 VOLUME $LMANIFEST_DIR
 VOLUME $DELTA_DIR
 VOLUME $KEYS_DIR
+VOLUME $LOGS_DIR

 # Copy required files
 #####################
@ -119,6 +124,7 @@ RUN mkdir -p $ZIP_DIR
 RUN mkdir -p $LMANIFEST_DIR
 RUN mkdir -p $DELTA_DIR
 RUN mkdir -p $KEYS_DIR
+RUN mkdir -p $LOGS_DIR

 # Install build dependencies
 ############################
--- a/README.md
+++ b/README.md
@ -106,11 +106,12 @@ docker run \
    -e "SIGN_BUILDS=true" \
    -e "CLEAN_OUTDIR=false" \
    -e "CLEAN_AFTER_BUILD=true" \
-    -e "DEBUG=true" \
    -e "ZIP_SUBDIR=true" \
+    -e "LOGS_SUBDIR=true" \
    -e "BUILD_DELTA=true" \
    -e "DELETE_OLD_ZIPS=3" \
    -e "DELETE_OLD_DELTAS=10" \
+    -e "DELETE_OLD_LOGS=10" \
    -e "OPENDELTA_BUILDS_JSON=builds.json" \
    -v "/home/user/ccache:/srv/ccache" \
    -v "/home/user/source:/srv/src" \
@ -118,6 +119,7 @@ docker run \
    -v "/home/user/local_manifests:/srv/local_manifests" \
    -v "/home/user/keys:/srv/keys" \
    -v "/home/user/public/delta:/srv/delta" \
+    -v "/home/user/logs:/srv/logs" \
    lineageos4microg/docker-lineage-cicd
 ```
 with the following XML in local_manifests
--- a/src/build.sh
+++ b/src/build.sh
@ -17,12 +17,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-DOCKER_LOG=/var/log/docker.log
-DEBUG_LOG=/dev/null
-if [ "$DEBUG" = true ]; then
-  DEBUG_LOG=$DOCKER_LOG
-fi
-
 if ! [ -z "$DEVICE_LIST" ]; then

  # cd to working directory
@ -31,32 +25,32 @@ if ! [ -z "$DEVICE_LIST" ]; then
  # If the source directory is empty
  if ! [ "$(ls -A $SRC_DIR)" ]; then
    # Initialize repository
-    echo ">> [$(date)] Initializing repository" >> $DOCKER_LOG
-    yes | repo init -u https://github.com/lineageos/android.git -b $BRANCH_NAME 2>&1 >&$DEBUG_LOG
+    echo ">> [$(date)] Initializing repository"
+    yes | repo init -u https://github.com/lineageos/android.git -b
  fi

  # Copy local manifests to the appropriate folder in order take them into consideration
-  echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '$SRC_DIR/.repo/local_manifests/'" >> $DOCKER_LOG
-  cp $LMANIFEST_DIR/*.xml $SRC_DIR/.repo/local_manifests/ >&$DEBUG_LOG
+  echo ">> [$(date)] Copying '$LMANIFEST_DIR/*.xml' to '$SRC_DIR/.repo/local_manifests/'"
+  rsync -a --delete --exclude 'roomservice.xml' --include '*.xml' --exclude '*' $LMANIFEST_DIR/ $SRC_DIR/.repo/local_manifests/

  # Reset the current git status of "vendor/cm" (remove previous changes) if the directory exists
  if [ -d "vendor/cm" ]; then
    cd vendor/cm
-    git reset --hard 2>&1 >&$DEBUG_LOG
+    git reset -q --hard
    cd $SRC_DIR
  fi

  # Reset the current git status of "frameworks/base" (remove previous changes) if the directory exists
  if [ -d "frameworks/base" ]; then
    cd frameworks/base
-    git reset --hard 2>&1 >&$DEBUG_LOG
+    git reset -q --hard
    cd $SRC_DIR
  fi

  # Sync the source code
-  echo ">> [$(date)] Syncing repository" >> $DOCKER_LOG
+  echo ">> [$(date)] Syncing repository"
  builddate=$(date +%Y%m%d)
-  repo sync 2>&1 >&$DEBUG_LOG
+  repo sync -q

  # If needed, apply the MicroG's signature spoofing patch
  if [ "$SIGNATURE_SPOOFING" = "yes" ] || [ "$SIGNATURE_SPOOFING" = "restricted" ]; then
@ -73,16 +67,16 @@ if ! [ -z "$DEVICE_LIST" ]; then
    if ! [ -z $patch_name ]; then
      cd frameworks/base
      if [ "$SIGNATURE_SPOOFING" = "yes" ]; then
-        echo ">> [$(date)] Applying the standard signature spoofing patch ($patch_name) to frameworks/base" >> $DOCKER_LOG
-        echo ">> [$(date)] WARNING: the standard signature spoofing patch introduces a security threat" >> $DOCKER_LOG
-        patch -p1 -i "/root/signature_spoofing_patches/$patch_name" 2>&1 >&$DEBUG_LOG
+        echo ">> [$(date)] Applying the standard signature spoofing patch ($patch_name) to frameworks/base"
+        echo ">> [$(date)] WARNING: the standard signature spoofing patch introduces a security threat"
+        patch --quiet -p1 -i "/root/signature_spoofing_patches/$patch_name"
      else
-        echo ">> [$(date)] Applying the restricted signature spoofing patch (based on $patch_name) to frameworks/base" >> $DOCKER_LOG
-        sed 's/android:protectionLevel="dangerous"/android:protectionLevel="signature|privileged"/' "/root/signature_spoofing_patches/$patch_name" | patch -p1 2>&1 >&$DEBUG_LOG
+        echo ">> [$(date)] Applying the restricted signature spoofing patch (based on $patch_name) to frameworks/base"
+        sed 's/android:protectionLevel="dangerous"/android:protectionLevel="signature|privileged"/' "/root/signature_spoofing_patches/$patch_name" | patch --quiet -p1
      fi
-      git clean -f 2>&1 >&$DEBUG_LOG
+      git clean -q -f
    else
-      echo ">> [$(date)] ERROR: can't find a suitable signature spoofing patch for the current Android version ($android_version)" >> $DOCKER_LOG
+      echo ">> [$(date)] ERROR: can't find a suitable signature spoofing patch for the current Android version ($android_version)"
      exit 1
    fi
  fi
@ -90,34 +84,34 @@ if ! [ -z "$DEVICE_LIST" ]; then

  # If requested, clean the OUT dir in order to avoid clutter
  if [ "$CLEAN_OUTDIR" = true ]; then
-    echo ">> [$(date)] Cleaning '$ZIP_DIR'" >> $DOCKER_LOG
+    echo ">> [$(date)] Cleaning '$ZIP_DIR'"
    cd $ZIP_DIR
    rm *
    cd $SRC_DIR
  fi

  # Prepare the environment
-  echo ">> [$(date)] Preparing build environment" >> $DOCKER_LOG
-  source build/envsetup.sh 2>&1 >&$DEBUG_LOG
+  echo ">> [$(date)] Preparing build environment"
+  source build/envsetup.sh > /dev/null

-  echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type" >> $DOCKER_LOG
+  echo ">> [$(date)] Setting \"$RELEASE_TYPE\" as release type"
  sed -i '/#.*Filter out random types/d' vendor/cm/config/common.mk
  sed -i '/$(filter .*$(CM_BUILDTYPE)/,+3d' vendor/cm/config/common.mk

  # Set a custom updater URI if a OTA URL is provided
  if ! [ -z "$OTA_URL" ]; then
-    echo ">> [$(date)] Adding OTA URL '$OTA_URL' to build.prop" >> $DOCKER_LOG
-    sed -i "1s;^;PRODUCT_PROPERTY_OVERRIDES += cm.updater.uri=$OTA_URL\n\n;" vendor/cm/config/common.mk >&$DEBUG_LOG
+    echo ">> [$(date)] Adding OTA URL '$OTA_URL' to build.prop"
+    sed -i "1s;^;PRODUCT_PROPERTY_OVERRIDES += cm.updater.uri=$OTA_URL\n\n;" vendor/cm/config/common.mk
  fi

  # Add custom packages to be installed
  if ! [ -z "$CUSTOM_PACKAGES" ]; then
-    echo ">> [$(date)] Adding custom packages ($CUSTOM_PACKAGES)" >> $DOCKER_LOG
+    echo ">> [$(date)] Adding custom packages ($CUSTOM_PACKAGES)"
    sed -i "1s;^;PRODUCT_PACKAGES += $CUSTOM_PACKAGES\n\n;" vendor/cm/config/common.mk
  fi

  if [ "$SIGN_BUILDS" = true ]; then
-    echo ">> [$(date)] Adding keys path ($KEYS_DIR)" >> $DOCKER_LOG
+    echo ">> [$(date)] Adding keys path ($KEYS_DIR)"
    sed -i "1s;^;PRODUCT_DEFAULT_DEV_CERTIFICATE := $KEYS_DIR/releasekey\nPRODUCT_OTA_PUBLIC_KEYS := $KEYS_DIR/releasekey\nPRODUCT_EXTRA_RECOVERY_KEYS := $KEYS_DIR/releasekey\n\n;" vendor/cm/config/common.mk
  fi

@ -127,9 +121,9 @@ if ! [ -z "$DEVICE_LIST" ]; then
    currentdate=$(date +%Y%m%d)
    if [ "$builddate" != "$currentdate" ]; then
      # Sync the source code
-      echo ">> [$(date)] Syncing repository" >> $DOCKER_LOG
+      echo ">> [$(date)] Syncing repository"
      builddate=$currentdate
-      repo sync 2>&1 >&$DEBUG_LOG
+      repo sync -q
    fi

    if ! [ -z "$codename" ]; then
@ -139,64 +133,77 @@ if ! [ -z "$DEVICE_LIST" ]; then
      else
        zipsubdir=
      fi
+      if [ "$LOGS_SUBDIR" = true ]; then
+        logsubdir=$codename
+        mkdir -p $LOGS_DIR/$logsubdir
+      else
+        logsubdir=
+      fi
+      los_ver_major=$(sed -n -e 's/^\s*PRODUCT_VERSION_MAJOR = //p' vendor/cm/config/common.mk)
+      los_ver_minor=$(sed -n -e 's/^\s*PRODUCT_VERSION_MINOR = //p' vendor/cm/config/common.mk)
+      DEBUG_LOG="$LOGS_DIR/$logsubdir/lineage-$los_ver_major.$los_ver_minor-$builddate-$RELEASE_TYPE-$codename.log"
      # Start the build
-      echo ">> [$(date)] Starting build for $codename" >> $DOCKER_LOG
-      if brunch $codename 2>&1 >&$DEBUG_LOG; then
+      echo ">> [$(date)] Starting build for $codename" | tee -a $DEBUG_LOG
+      if brunch $codename >> $DEBUG_LOG 2>&1; then
        currentdate=$(date +%Y%m%d)
        if [ "$builddate" != "$currentdate" ]; then
-          find out/target/product/$codename -name "lineage-*-$currentdate-*.zip*" -exec sh /root/fix_build_date.sh {} $currentdate $builddate \;
+          find out/target/product/$codename -name "lineage-*-$currentdate-*.zip*" -exec sh /root/fix_build_date.sh {} $currentdate $builddate \; >> $DEBUG_LOG 2>&1
        fi

        if [ "$BUILD_DELTA" = true ]; then
          if [ -d "$SRC_DIR/delta_last/$codename/" ]; then
            # If not the first build, create delta files
-            echo ">> [$(date)] Generating delta files for $codename" >> $DOCKER_LOG
+            echo ">> [$(date)] Generating delta files for $codename" | tee -a $DEBUG_LOG
            cd /root/delta
-            if ./opendelta.sh $codename >&$DEBUG_LOG; then
-              echo ">> [$(date)] Delta generation for $codename completed" >> $DOCKER_LOG
+            if ./opendelta.sh $codename >> $DEBUG_LOG 2>&1; then
+              echo ">> [$(date)] Delta generation for $codename completed" | tee -a $DEBUG_LOG
            else
-              echo ">> [$(date)] Delta generation for $codename failed" >> $DOCKER_LOG
+              echo ">> [$(date)] Delta generation for $codename failed" | tee -a $DEBUG_LOG
            fi
            if [ "$DELETE_OLD_DELTAS" -gt "0" ]; then
-              /usr/bin/python /root/clean_up.py -n $DELETE_OLD_DELTAS $DELTA_DIR
+              /usr/bin/python /root/clean_up.py -n $DELETE_OLD_DELTAS $DELTA_DIR >> $DEBUG_LOG 2>&1
            fi
          else
            # If the first build, copy the current full zip in $SRC_DIR/delta_last/$codename/
-            echo ">> [$(date)] No previous build for $codename; using current build as base for the next delta" >> $DOCKER_LOG
-            mkdir -p $SRC_DIR/delta_last/$codename/
-            find out/target/product/$codename -name 'lineage-*.zip' -exec cp {} $SRC_DIR/delta_last/$codename/ \;
+            echo ">> [$(date)] No previous build for $codename; using current build as base for the next delta" | tee -a $DEBUG_LOG
+            mkdir -p $SRC_DIR/delta_last/$codename/ >> $DEBUG_LOG 2>&1
+            find out/target/product/$codename -name 'lineage-*.zip' -exec cp {} $SRC_DIR/delta_last/$codename/ \; >> $DEBUG_LOG 2>&1
          fi
        fi
        # Move produced ZIP files to the main OUT directory
-        echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" >> $DOCKER_LOG
+        echo ">> [$(date)] Moving build artifacts for $codename to '$ZIP_DIR/$zipsubdir'" | tee -a $DEBUG_LOG
        cd $SRC_DIR
-        find out/target/product/$codename -name 'lineage-*.zip*' -exec mv {} $ZIP_DIR/$zipsubdir/ \; >&$DEBUG_LOG
-        if [ "$DELETE_OLD_ZIPS" -gt "0" ]; then
-          /usr/bin/python /root/clean_up.py -n $DELETE_OLD_ZIPS $ZIP_DIR
-        fi
+        find out/target/product/$codename -name 'lineage-*.zip*' -exec mv {} $ZIP_DIR/$zipsubdir/ \; >> $DEBUG_LOG 2>&1
      else
-        echo ">> [$(date)] Failed build for $codename" >> $DOCKER_LOG
+        echo ">> [$(date)] Failed build for $codename" | tee -a $DEBUG_LOG
+      fi
+      # Remove old zips and logs
+      if [ "$DELETE_OLD_ZIPS" -gt "0" ]; then
+        /usr/bin/python /root/clean_up.py -n $DELETE_OLD_ZIPS $ZIP_DIR
+      fi
+      if [ "$DELETE_OLD_LOGS" -gt "0" ]; then
+        /usr/bin/python /root/clean_up.py -n $DELETE_OLD_LOGS $LOGS_DIR
      fi
      # Clean everything, in order to start fresh on next build
      if [ "$CLEAN_AFTER_BUILD" = true ]; then
-        echo ">> [$(date)] Cleaning build for $codename" >> $DOCKER_LOG
-        rm -rf $SRC_DIR/out/target/product/$codename/
+        echo ">> [$(date)] Cleaning build for $codename" | tee -a $DEBUG_LOG
+        rm -rf $SRC_DIR/out/target/product/$codename/ >> $DEBUG_LOG 2>&1
      fi
-      echo ">> [$(date)] Finishing build for $codename" >> $DOCKER_LOG
+      echo ">> [$(date)] Finishing build for $codename" | tee -a $DEBUG_LOG
    fi
  done

  # Create the OpenDelta's builds JSON file
  if ! [ -z "$OPENDELTA_BUILDS_JSON" ]; then
-    echo ">> [$(date)] Creating OpenDelta's builds JSON file (ZIP_DIR/$OPENDELTA_BUILDS_JSON)" >> $DOCKER_LOG
+    echo ">> [$(date)] Creating OpenDelta's builds JSON file (ZIP_DIR/$OPENDELTA_BUILDS_JSON)"
    if [ "$ZIP_SUBDIR" != true ]; then
-      echo ">> [$(date)] WARNING: OpenDelta requires zip builds separated per device! You should set ZIP_SUBDIR to true" >> $DOCKER_LOG
+      echo ">> [$(date)] WARNING: OpenDelta requires zip builds separated per device! You should set ZIP_SUBDIR to true"
    fi
    /usr/bin/python /root/opendelta_builds_json.py $ZIP_DIR -o $ZIP_DIR/$OPENDELTA_BUILDS_JSON
  fi

  # Clean the src directory if requested
  if [ "$CLEAN_SRCDIR" = true ]; then
-    rm -Rf "$SRC_DIR/*"
+    rm -rf "$SRC_DIR/*"
  fi
 fi
--- a/src/init.sh
+++ b/src/init.sh
@ -17,15 +17,9 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.

-DOCKER_LOG=/var/log/docker.log
-DEBUG_LOG=/dev/null
-if [ "$DEBUG" = true ]; then
-  DEBUG_LOG=$DOCKER_LOG
-fi
-
 # Initialize CCache if it will be used
 if [ "$USE_CCACHE" = 1 ]; then
-  ccache -M 100G 2>&1 >&$DEBUG_LOG
+  ccache -M 100G 2>&1
 fi

 # Initialize Git user information