Merge remote-tracking branch 'remotes/tjburrows/master' into lineage-20-python

.github/workflows/pytest.yml

@@ -0,0 +1,18 @@
+name: pytest
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  pytest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Build docker container
+        run: docker build -t docker-lineage-cicd-temp .
+
+      - name: Run tests
+        run: docker run --entrypoint pytest-3 docker-lineage-cicd-temp /test

Dockerfile

@@ -146,8 +146,8 @@ RUN apt-get -qq update && \
       cron curl flex g++-multilib gcc-multilib git gnupg gperf imagemagick \
       kmod lib32ncurses5-dev lib32readline-dev lib32z1-dev liblz4-tool \
       libncurses5 libncurses5-dev libsdl1.2-dev libssl-dev libxml2 \
-      libxml2-utils lsof lzop maven openjdk-8-jdk pngcrush procps python3 \
-      python-is-python3 rsync schedtool squashfs-tools wget xdelta3 xsltproc yasm zip \
+      libxml2-utils lsof lzop maven openjdk-8-jdk pngcrush procps python3 python3-apscheduler \
+      python3-pytest python-is-python3  rsync schedtool squashfs-tools wget xdelta3 xsltproc yasm zip \
       zlib1g-dev \
       && rm -rf /var/lib/apt/lists/*
 
@@ -162,6 +162,7 @@ RUN echo "jdk.tls.disabledAlgorithms=SSLv3, RC4, DES, MD5withRSA, DH keySize < 1
 # Copy required files
 #####################
 COPY src/ /root/
+COPY test/ /test/
 
 # Set the work directory
 ########################
@@ -173,4 +174,4 @@ RUN ln -sf /proc/1/fd/1 /var/log/docker.log
 
 # Set the entry point to init.sh
 ################################
-ENTRYPOINT /root/init.sh
+ENTRYPOINT ["python", "/root/init.py"]

src/build.py

@@ -0,0 +1,7 @@
+import subprocess
+
+def build() -> None:
+    subprocess.run(["/root/build.sh"], check=True, stderr=subprocess.STDOUT)
+
+if __name__ == "__main__":
+    build()

src/init.py

@@ -0,0 +1,157 @@
+from os import getenv
+import shutil
+import subprocess
+from pathlib import Path
+from itertools import product
+import build
+from apscheduler.schedulers.blocking import BlockingScheduler
+from apscheduler.triggers.cron import CronTrigger
+import logging
+import sys
+
+
+def getvar(var: str) -> str:
+    val = getenv(var)
+    if val == "" or val is None:
+        raise ValueError('Environment variable "%s" has an invalid value.' % var)
+    return val
+
+
+def make_key(key_path: str, key_subj: str) -> None:
+    subprocess.run(
+        ["/root/make_key", key_path, key_subj],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        check=True,
+        input="\n".encode(),
+    )
+
+
+class Init:
+    def __init__(self) -> None:
+        self.root_scripts = "/root/user_scripts"
+        self.user_scripts = getvar("USERSCRIPTS_DIR")
+        self.use_ccache = getvar("USE_CCACHE").lower() in ["1", "true"]
+        self.sign_builds = getvar("SIGN_BUILDS").lower() == "true"
+        if self.sign_builds:
+            self.key_dir = Path(getvar("KEYS_DIR"))
+        if self.use_ccache:
+            self.ccache_size = getvar("CCACHE_SIZE")
+        self.cron_time = getvar("CRONTAB_TIME")
+        self.git_username = getvar("USER_NAME")
+        self.git_email = getvar("USER_MAIL")
+        self.key_subj = getvar("KEYS_SUBJECT")
+        self.key_names = [
+            "releasekey",
+            "platform",
+            "shared",
+            "media",
+            "networkstack",
+            "sdk_sandbox",
+            "bluetooth",
+        ]
+        self.key_exts = [".pk8", ".x509.pem"]
+        self.key_aliases = ["cyngn-priv-app", "cyngn-app", "testkey"]
+        # New keys needed as of LOS20
+        self.new_key_names = [
+            "sdk_sandbox",
+            "bluetooth",
+        ]
+
+        logging.basicConfig(
+            stream=sys.stdout,
+            level=logging.INFO,
+            format="[%(asctime)s] %(levelname)s %(message)s",
+            datefmt="%c %Z",
+        )
+
+    def generate_key(self, key_name: str) -> None:
+        logging.info("Generating %s..." % key_name)
+        make_key(str(self.key_dir.joinpath(key_name)), self.key_subj)
+
+    def do(self) -> None:
+        # Copy the user scripts
+        shutil.copytree(self.user_scripts, self.root_scripts)
+
+        # Delete non-root files
+        to_delete = []
+        for path in Path(self.root_scripts).rglob("*"):
+            if path.is_dir():
+                continue
+
+            # Check if not owned by root
+            if path.owner != "root":
+                logging.warning("File not owned by root. Removing %s", path)
+                to_delete.append(path)
+                continue
+
+            # Check if non-root can write (group or other)
+            perm = oct(path.stat().st_mode)
+            group_write = perm[-2] > "4"
+            other_write = perm[-1] > "4"
+            if group_write or other_write:
+                logging.warning("File writable by non root users. Removing %s", path)
+                to_delete.append(path)
+
+        for f in to_delete:
+            f.unlink()
+
+        # Initialize CCache if it will be used
+        if self.use_ccache:
+            subprocess.run(
+                ["ccache", "-M", self.ccache_size], check=True, stderr=subprocess.STDOUT
+            )
+
+        # Initialize Git user information
+        subprocess.run(
+            ["git", "config", "--global", "user.name", self.git_username], check=True
+        )
+        subprocess.run(
+            ["git", "config", "--global", "user.email", self.git_email], check=True
+        )
+
+        if self.sign_builds:
+            # Generate keys if directory empty
+            if not list(self.key_dir.glob("*")):
+                logging.info(
+                    "SIGN_BUILDS = true but empty $KEYS_DIR, generating new keys"
+                )
+                for k in self.key_names:
+                    self.generate_key(k)
+
+            # Check that all expected key files exist.  If a LOS20 key does not exist, create it.
+            for k, e in product(self.key_names, self.key_exts):
+                path = self.key_dir.joinpath(k).with_suffix(e)
+                if not path.exists():
+                    if k in self.new_key_names:
+                        self.generate_key(k)
+                    else:
+                        raise AssertionError(
+                            'Expected key file "%s" does not exist' % path
+                        )
+
+            # Create releasekey aliases
+            for a, e in product(self.key_aliases, self.key_exts):
+                src = self.key_dir.joinpath("releasekey").with_suffix(e)
+                dst = self.key_dir.joinpath(a).with_suffix(e)
+                dst.symlink_to(src)
+
+        if self.cron_time == "now":
+            build.build()
+        else:
+            scheduler = BlockingScheduler()
+            scheduler.add_job(
+                func=build.build,
+                trigger=CronTrigger.from_crontab(self.cron_time),
+                misfire_grace_time=None,  # Allow job to run as long as it needs
+                coalesce=True,
+                max_instances=1,  # Allow only one concurrent instance
+            )
+
+            # Run forever
+            scheduler.start()
+
+
+if __name__ == "__main__":
+    initialize = Init()
+    initialize.do()

src/init.sh

@@ -1,83 +0,0 @@
-#!/bin/bash
-
-# Docker init script
-# Copyright (c) 2017 Julian Xhokaxhiu
-# Copyright (C) 2017-2018 Nicola Corna <nicola@corna.info>
-# 
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-set -eEuo pipefail
-
-# Copy the user scripts
-mkdir -p /root/userscripts
-cp -r "$USERSCRIPTS_DIR"/. /root/userscripts
-find /root/userscripts ! -type d ! -user root -exec echo ">> [$(date)] {} is not owned by root, removing" \; -exec rm {} \;
-find /root/userscripts ! -type d -perm /g=w,o=w -exec echo ">> [$(date)] {} is writable by non-root users, removing" \; -exec rm {} \;
-
-# Initialize CCache if it will be used
-if [ "$USE_CCACHE" = 1 ]; then
-  ccache -M "$CCACHE_SIZE" 2>&1
-fi
-
-# Initialize Git user information
-git config --global user.name "$USER_NAME"
-git config --global user.email "$USER_MAIL"
-
-if [ "$SIGN_BUILDS" = true ]; then
-  if [ -z "$(ls -A "$KEYS_DIR")" ]; then
-    echo ">> [$(date)] SIGN_BUILDS = true but empty \$KEYS_DIR, generating new keys"
-    for c in releasekey platform shared media networkstack sdk_sandbox bluetooth; do
-      echo ">> [$(date)]  Generating $c..."
-      /root/make_key "$KEYS_DIR/$c" "$KEYS_SUBJECT" <<< '' &> /dev/null
-    done
-  else
-    for c in releasekey platform shared media networkstack; do
-      for e in pk8 x509.pem; do
-        if [ ! -f "$KEYS_DIR/$c.$e" ]; then
-          echo ">> [$(date)] SIGN_BUILDS = true and not empty \$KEYS_DIR, but \"\$KEYS_DIR/$c.$e\" is missing"
-          exit 1
-        fi
-      done
-    done
-    
-    # those keys are only required starting with android-20, so people who have built earlier might not yet have them
-    for c in sdk_sandbox bluetooth; do
-      if [ ! -f "$KEYS_DIR/$c.pk8" ]; then
-        echo ">> [$(date)]  Generating $c..."
-        /root/make_key "$KEYS_DIR/$c" "$KEYS_SUBJECT" <<< '' &> /dev/null
-      fi
-    done
-  fi
-
-  for c in cyngn{-priv,}-app testkey; do
-    for e in pk8 x509.pem; do
-      ln -sf releasekey.$e "$KEYS_DIR/$c.$e" 2> /dev/null
-    done
-  done
-fi
-
-if [ "$CRONTAB_TIME" = "now" ]; then
-  /root/build.sh
-else
-  # Initialize the cronjob
-  cronFile=/tmp/buildcron
-  printf "SHELL=/bin/bash\n" > $cronFile
-  printenv -0 | sed -e 's/=\x0/=""\n/g'  | sed -e 's/\x0/\n/g' | sed -e "s/_=/PRINTENV=/g" >> $cronFile
-  printf '\n%s /usr/bin/flock -n /var/lock/build.lock /root/build.sh >> /var/log/docker.log 2>&1\n' "$CRONTAB_TIME" >> $cronFile
-  crontab $cronFile
-  rm $cronFile
-
-  # Run crond in foreground
-  cron -f 2>&1
-fi

test/init_test.py

@@ -0,0 +1,23 @@
+import sys
+from itertools import product
+from pathlib import Path
+
+sys.path.append("/root")
+import build
+import init
+
+
+def test_key_gen(monkeypatch):
+    def mock_build() -> None:
+        print("mock build")
+
+    monkeypatch.setenv("SIGN_BUILDS", "true")
+    monkeypatch.setattr(build, "build", mock_build)
+
+    initialize = init.Init()
+    initialize.do()
+
+    # Confirm all keys are generated
+    for k, e in product(initialize.key_names, initialize.key_exts):
+        path = Path("/srv/keys").joinpath(k).with_suffix(e)
+        assert path.exists()