From 5cf892c7ea3b525285caf45e1a2ca638cca282f7 Mon Sep 17 00:00:00 2001 From: surtur Date: Thu, 3 Nov 2022 21:18:55 +0100 Subject: [PATCH] initial commit --- .editorconfig | 28 +++++++ .envrc | 7 ++ .gitattributes | 1 + .gitignore | 97 +++++++++++++++++++++++++ default.nix | 16 ++++ flake.lock | 44 +++++++++++ flake.nix | 102 ++++++++++++++++++++++++++ p1/README.md | 3 + p1/p1/__init__.py | 0 p1/p1/funcs.py | 169 +++++++++++++++++++++++++++++++++++++++++++ p1/p1/main.py | 72 ++++++++++++++++++ p1/poetry.lock | 8 ++ p1/pyproject.toml | 14 ++++ p1/tests/__init__.py | 0 p1/tox.ini | 6 ++ shell.nix | 16 ++++ 16 files changed, 583 insertions(+) create mode 100644 .editorconfig create mode 100644 .envrc create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 default.nix create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 p1/README.md create mode 100644 p1/p1/__init__.py create mode 100644 p1/p1/funcs.py create mode 100644 p1/p1/main.py create mode 100644 p1/poetry.lock create mode 100644 p1/pyproject.toml create mode 100644 p1/tests/__init__.py create mode 100644 p1/tox.ini create mode 100644 shell.nix diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..8c7a916 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,28 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf + +# starlark +[*.star] +indent_style = tab +indent_size = 4 +tab_width = 4 + +# go files (excluding templates) +[*.{go,mod,sum}] +indent_style = tab +indent_size = 4 +tab_width = 4 + +[*.nix] +indent_style = space +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false + +[*.{yaml,yml}] +indent_style = space +indent_size = 2 diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..6125787 --- /dev/null +++ b/.envrc @@ -0,0 +1,7 @@ +use flake + +# ref: https://github.com/direnv/direnv/wiki/Vim +# comment out if not planning to use this +#add_extra_vimrc + +# vim: ff=unix ft=sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6889ef2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.md diff=markdown diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ccdfd7b --- /dev/null +++ b/.gitignore @@ -0,0 +1,97 @@ +.direnv + +# binaries/symlinks to binaries +result +result-* + +# generated from starlark +.drone.yml + +.vimrc + +# coverage or other binary files +*.out + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..c580745 --- /dev/null +++ b/default.nix @@ -0,0 +1,16 @@ +( + import + ( + let + lock = builtins.fromJSON (builtins.readFile ./flake.lock); + in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { + src = ./.; + } +) +.defaultNix diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..eadaf3b --- /dev/null +++ b/flake.lock @@ -0,0 +1,44 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1650374568, + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1666244578, + "narHash": "sha256-OO0F2b83isMVHYtcvxiExig28zPE7Weo7r1fHtQTZzU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "260eb420a2e55e3a0411e731b933c3a8bf6b778e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..b0841fd --- /dev/null +++ b/flake.nix @@ -0,0 +1,102 @@ +{ + description = "Identification and modeling of stochastic signals - protocols"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-compat = { + url = "github:edolstra/flake-compat"; + flake = false; + }; + }; + + outputs = { + self, + nixpkgs, + ... + }: let + projname = "ak9im"; + + # to work with older version of flakes + lastModifiedDate = + self.lastModifiedDate or self.lastModified or "19700101"; + + # Generate a user-friendly version number. + version = "v0.0.0"; + + supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + pkgs = forAllSystems (system: nixpkgs.legacyPackages.${system}); + # Nixpkgs instantiated for supported system types. + nixpkgsFor = forAllSystems (system: + import nixpkgs { + inherit system; + overlays = [ + # no overlay imports atm + # (import ./overlay.nix) + ]; + }); + in { + formatter = forAllSystems ( + system: + nixpkgsFor.${system}.alejandra + ); + packages = forAllSystems (system: let + baseurl = "https://git.dotya.ml/wanderer/ak9im/"; + in rec { + p1 = pkgs.${system}.poetry2nix.mkPoetryApplication { + name = "p1"; + projectDir = ./p1; + meta = { + description = self.description + ": p1"; + homepage = baseurl + "p1"; + license = nixpkgs.lib.licenses.gpl3; + maintainers = ["wanderer"]; + platforms = nixpkgs.lib.platforms.linux ++ nixpkgs.lib.platforms.darwin; + }; + }; + default = p1; + }); + + apps = forAllSystems (system: rec { + p1 = { + type = "app"; + program = "${self.packages.${system}.${projname}}/bin/p1"; + }; + default = p1; + }); + + devShells = forAllSystems (system: let + upcache = pkgs.writeShellScriptBin "upcache" '' + ## refs: + ## https://fzakaria.com/2020/08/11/caching-your-nix-shell.html + ## https://nixos.wiki/wiki/Caching_nix_shell_build_inputs + nix-store --query --references $(nix-instantiate shell.nix) | \ + xargs nix-store --realise | \ + xargs nix-store --query --requisites | \ + cachix push ${projname} + nix build --json \ + | jq -r '.[].outputs | to_entries[].value' \ + | cachix push ${projname} + ''; + add-license = pkgs.writeShellScriptBin "add-license" '' + go run github.com/google/addlicense@v1.0.0 -v \ + -c "wanderer " \ + -l "GPL-3.0-or-later" -s . + ''; + in { + default = pkgs.${system}.mkShellNoCC { + name = "${projname}-" + version; + shellHook = '' + echo " -- in ${projname} dev shell..." + ''; + + packages = with pkgs.${system}; [ + (poetry2nix.mkPoetryEnv {projectDir = ./p1;}) + python3Packages.numpy + python3Packages.pandas + python3Packages.matplotlib + poetry + ]; + }; + }); + }; +} diff --git a/p1/README.md b/p1/README.md new file mode 100644 index 0000000..d3d3d69 --- /dev/null +++ b/p1/README.md @@ -0,0 +1,3 @@ +# p1 + +this subproject contains code for protocol no. 1. diff --git a/p1/p1/__init__.py b/p1/p1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/p1/p1/funcs.py b/p1/p1/funcs.py new file mode 100644 index 0000000..2fc21c2 --- /dev/null +++ b/p1/p1/funcs.py @@ -0,0 +1,169 @@ +import pandas as pd +import numpy as np +import matplotlib.pyplot as pyplt + + +def plt_ticks_size() -> int: + return 10 + + +def load_d(path: str) -> pd.DataFrame(): + return pd.read_csv(path, float_precision='round_trip', dtype='float64') + + +def plot_d(dat: pd.Series, fname: str = 'plot_input_data', colour: str = ''): + if colour == '': + colour = 'blue' + + pyplt.plot(dat, color=colour) + pyplt.xlim(0, len(dat)) + pyplt.ylim(min(dat) - 0.3, max(dat) + 0.3) + pyplt.xticks(size=plt_ticks_size()) + pyplt.yticks(size=plt_ticks_size()) + pyplt.savefig(fname + '.jpg') + # pplt.show(block=0) + # rework this + # ref: https://stackoverflow.com/a/46418284 + pyplt.close() + + +def mean(dat: pd.Series) -> float: + return dat.sum() / len(dat) + + +def variance(dat: pd.Series) -> float: + return sum(pow(dat - mean(dat), 2)) / len(dat) + + +def histogram(dat: pd.Series, bins: int = 10, fname: str = 'hist', colour: str = 'blue'): + pyplt.hist(dat, color=colour, bins=bins) + pyplt.xticks(size=plt_ticks_size()) + pyplt.yticks(size=plt_ticks_size()) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def distr_func(dat: pd.Series, fname: str = 'dist', colour: str = 'blue'): + dat.plot.density(color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def std_dev(dat: pd.Series) -> float: + return np.sqrt(variance(dat)) + + +def mean_diff(dat: pd.DataFrame) -> pd.Series: + return dat - dat.mean() + + +def covar(dat: pd.DataFrame) -> float: + dat = dat.apply(mean_diff) + return dat['u'].dot(dat['y']) / len(dat) + + +def covar_coeff(cov: float, std_dev_u: float, std_dev_y: float) -> float: + return cov / (std_dev_u * std_dev_y) + + +# m is the max permissible shift value. +def auto_corellation(dat: list, max_shift_n: int = .1) -> float: + v = [] + m = len(dat) * max_shift_n + cur_shift = 0 + + while m >= cur_shift: + r = 0 + + for i in range(len(dat) - cur_shift): + r += dat[i] * dat[i + cur_shift] + + r = r / len(dat) - cur_shift + v.append(r) + cur_shift += 1 + + return v + + +def mutual_corellation(dx: list, dy: list, max_shift_n: int = .1) -> float: + v = [] + m = len(dx) * max_shift_n + cur_shift = 0 + + while m >= cur_shift: + r = 0 + + for i in range(len(dx) - cur_shift): + r += dx[i] * dy[i + cur_shift] + + r = r / (len(dx) - cur_shift) + v.append(r) + cur_shift += 1 + + return v + + +def auto_covar(dat: list, max_shift_n: int = .1) -> float: + v = [] + mean = np.mean(dat) + m = len(dat) * max_shift_n + cur_shift = 0 + + while m >= cur_shift: + r = 0 + + for i in range(len(dat) - cur_shift): + r += (dat[i] - mean) * (dat[i + cur_shift] - mean) + + r = r / (len(dat) - cur_shift) + v.append(r) + cur_shift += 1 + + return v + + +def mutual_covar(dx: list, dy: list, max_shift_n: int = .1) -> float: + v = [] + mean_x, mean_y = np.mean(dx), np.mean(dy) + m = len(dx) * max_shift_n + cur_shift = 0 + + while m >= cur_shift: + r = 0 + + for i in range(len(dx) - cur_shift): + r += (dx[i] - mean_x) * (dy[i + cur_shift] - mean_y) + + r = r / (len(dx) - cur_shift) + v.append(r) + cur_shift += 1 + + return v + + +def plot_autocorellation(dat: pd.DataFrame, fname: str = 'autocorellation', colour: str = 'blue'): + d = auto_corellation(dat.tolist()) + pyplt.scatter(range(0, len(d)), d, color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def plot_mutual_corellation(d1: pd.DataFrame, d2: pd.DataFrame, fname: str = 'mutcorellation', colour: str = 'blue'): + d = mutual_corellation(d1.tolist(), d2.tolist()) + pyplt.scatter(range(0, len(d)), d, color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def plot_autocovariance(dat: pd.DataFrame, fname: str = 'autocovariance', colour: str = 'blue'): + d = auto_covar(dat.tolist()) + pyplt.scatter(range(0, len(d)), d, color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def plot_mutual_covariance(d1: pd.DataFrame, d2: pd.DataFrame, fname: str = 'mutcovariance', colour: str = 'blue'): + d = mutual_covar(d1.tolist(), d2.tolist()) + pyplt.scatter(range(0, len(d)), d, color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() diff --git a/p1/p1/main.py b/p1/p1/main.py new file mode 100644 index 0000000..1e4ea04 --- /dev/null +++ b/p1/p1/main.py @@ -0,0 +1,72 @@ +import sys +import getopt +import numpy as np +import funcs as f + + +def main(argv): + inputfile = '' + try: + opts, args = getopt.getopt(argv, "hi:o:", ["ifile="]) + except getopt.GetoptError: + print('main.py -i ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print('main.py -i ') + sys.exit() + elif opt in ("-i", "--ifile"): + inputfile = arg + if inputfile != '': + print('Input file is', inputfile) + else: + print('no input file provided, see help (-h)\nexiting...') + sys.exit(1) + + # d = f.load_d("./dat.csv") + d = f.load_d(inputfile) + # d = d.astype('float64') + print(d.head()) + + # data plots + du = d['u'] + dy = d['y'] + f.plot_d(du, fname='u_input_plot') + f.plot_d(dy, fname='y_input_plot', colour='green') + + # mean and variance + mean_u = f.mean(d['u']) + mean_y = f.mean(d['y']) + + variance_u = f.variance(d['u']) + variance_y = f.variance(d['y']) + + hist_u = f.histogram(d['u'], fname='u_hist') + hist_y = f.histogram(d['y'], fname='y_hist', colour='green') + + dist_u = f.distr_func(d['u'], fname='u_dist') + dist_y = f.distr_func(d['y'], fname='y_dist', colour='green') + + cov = f.covar(d) + std_dev_u = f.std_dev(d['u']) + std_dev_y = f.std_dev(d['y']) + cov_c = f.covar_coeff(cov, std_dev_u, std_dev_y) + + print("data covariance:\n", d.cov()) + + # print the matrix. + print(np.array([[variance_u, cov], [cov, variance_y]])) + + f.plot_autocorellation(dat=d['u'], fname='u_autocorellation') + f.plot_autocorellation(dat=d['y'], fname='y_autocorellation', colour='green') + + f.plot_mutual_corellation(d1=d['u'], d2=d['y'], fname='mutual_corellation_uy') + + f.plot_autocovariance(dat=d['u'], fname='u_autocovariance') + f.plot_autocovariance(dat=d['y'], fname='y_autocovariance', colour='green') + + f.plot_mutual_covariance(d1=d['u'], d2=d['y'], fname='mutual_covariance_uy') + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/p1/poetry.lock b/p1/poetry.lock new file mode 100644 index 0000000..2ddd539 --- /dev/null +++ b/p1/poetry.lock @@ -0,0 +1,8 @@ +package = [] + +[metadata] +lock-version = "1.1" +python-versions = "^3.10" +content-hash = "53f2eabc9c26446fbcc00d348c47878e118afc2054778c3c803a0a8028af27d9" + +[metadata.files] diff --git a/p1/pyproject.toml b/p1/pyproject.toml new file mode 100644 index 0000000..c5deda5 --- /dev/null +++ b/p1/pyproject.toml @@ -0,0 +1,14 @@ +[tool.poetry] +name = "p1" +version = "0.0.0" +description = "" +authors = ["surtur "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/p1/tests/__init__.py b/p1/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/p1/tox.ini b/p1/tox.ini new file mode 100644 index 0000000..0d7b5db --- /dev/null +++ b/p1/tox.ini @@ -0,0 +1,6 @@ +[flake8] +indent_char='tab' +extend-ignore = W191, E501 +; ignore = +; # tabs are ok +; W191 diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..017cf3f --- /dev/null +++ b/shell.nix @@ -0,0 +1,16 @@ +( + import + ( + let + lock = builtins.fromJSON (builtins.readFile ./flake.lock); + in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { + src = ./.; + } +) +.shellNix