commit 5cf892c7ea3b525285caf45e1a2ca638cca282f7 Author: surtur Date: Thu Nov 3 21:18:55 2022 +0100 initial commit diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..8c7a916 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,28 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf + +# starlark +[*.star] +indent_style = tab +indent_size = 4 +tab_width = 4 + +# go files (excluding templates) +[*.{go,mod,sum}] +indent_style = tab +indent_size = 4 +tab_width = 4 + +[*.nix] +indent_style = space +indent_size = 2 + +[*.md] +trim_trailing_whitespace = false + +[*.{yaml,yml}] +indent_style = space +indent_size = 2 diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..6125787 --- /dev/null +++ b/.envrc @@ -0,0 +1,7 @@ +use flake + +# ref: https://github.com/direnv/direnv/wiki/Vim +# comment out if not planning to use this +#add_extra_vimrc + +# vim: ff=unix ft=sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6889ef2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.md diff=markdown diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ccdfd7b --- /dev/null +++ b/.gitignore @@ -0,0 +1,97 @@ +.direnv + +# binaries/symlinks to binaries +result +result-* + +# generated from starlark +.drone.yml + +.vimrc + +# coverage or other binary files +*.out + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..c580745 --- /dev/null +++ b/default.nix @@ -0,0 +1,16 @@ +( + import + ( + let + lock = builtins.fromJSON (builtins.readFile ./flake.lock); + in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { + src = ./.; + } +) +.defaultNix diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..eadaf3b --- /dev/null +++ b/flake.lock @@ -0,0 +1,44 @@ +{ + "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1650374568, + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1666244578, + "narHash": "sha256-OO0F2b83isMVHYtcvxiExig28zPE7Weo7r1fHtQTZzU=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "260eb420a2e55e3a0411e731b933c3a8bf6b778e", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..b0841fd --- /dev/null +++ b/flake.nix @@ -0,0 +1,102 @@ +{ + description = "Identification and modeling of stochastic signals - protocols"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + flake-compat = { + url = "github:edolstra/flake-compat"; + flake = false; + }; + }; + + outputs = { + self, + nixpkgs, + ... + }: let + projname = "ak9im"; + + # to work with older version of flakes + lastModifiedDate = + self.lastModifiedDate or self.lastModified or "19700101"; + + # Generate a user-friendly version number. + version = "v0.0.0"; + + supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; + forAllSystems = nixpkgs.lib.genAttrs supportedSystems; + pkgs = forAllSystems (system: nixpkgs.legacyPackages.${system}); + # Nixpkgs instantiated for supported system types. + nixpkgsFor = forAllSystems (system: + import nixpkgs { + inherit system; + overlays = [ + # no overlay imports atm + # (import ./overlay.nix) + ]; + }); + in { + formatter = forAllSystems ( + system: + nixpkgsFor.${system}.alejandra + ); + packages = forAllSystems (system: let + baseurl = "https://git.dotya.ml/wanderer/ak9im/"; + in rec { + p1 = pkgs.${system}.poetry2nix.mkPoetryApplication { + name = "p1"; + projectDir = ./p1; + meta = { + description = self.description + ": p1"; + homepage = baseurl + "p1"; + license = nixpkgs.lib.licenses.gpl3; + maintainers = ["wanderer"]; + platforms = nixpkgs.lib.platforms.linux ++ nixpkgs.lib.platforms.darwin; + }; + }; + default = p1; + }); + + apps = forAllSystems (system: rec { + p1 = { + type = "app"; + program = "${self.packages.${system}.${projname}}/bin/p1"; + }; + default = p1; + }); + + devShells = forAllSystems (system: let + upcache = pkgs.writeShellScriptBin "upcache" '' + ## refs: + ## https://fzakaria.com/2020/08/11/caching-your-nix-shell.html + ## https://nixos.wiki/wiki/Caching_nix_shell_build_inputs + nix-store --query --references $(nix-instantiate shell.nix) | \ + xargs nix-store --realise | \ + xargs nix-store --query --requisites | \ + cachix push ${projname} + nix build --json \ + | jq -r '.[].outputs | to_entries[].value' \ + | cachix push ${projname} + ''; + add-license = pkgs.writeShellScriptBin "add-license" '' + go run github.com/google/addlicense@v1.0.0 -v \ + -c "wanderer " \ + -l "GPL-3.0-or-later" -s . + ''; + in { + default = pkgs.${system}.mkShellNoCC { + name = "${projname}-" + version; + shellHook = '' + echo " -- in ${projname} dev shell..." + ''; + + packages = with pkgs.${system}; [ + (poetry2nix.mkPoetryEnv {projectDir = ./p1;}) + python3Packages.numpy + python3Packages.pandas + python3Packages.matplotlib + poetry + ]; + }; + }); + }; +} diff --git a/p1/README.md b/p1/README.md new file mode 100644 index 0000000..d3d3d69 --- /dev/null +++ b/p1/README.md @@ -0,0 +1,3 @@ +# p1 + +this subproject contains code for protocol no. 1. diff --git a/p1/p1/__init__.py b/p1/p1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/p1/p1/funcs.py b/p1/p1/funcs.py new file mode 100644 index 0000000..2fc21c2 --- /dev/null +++ b/p1/p1/funcs.py @@ -0,0 +1,169 @@ +import pandas as pd +import numpy as np +import matplotlib.pyplot as pyplt + + +def plt_ticks_size() -> int: + return 10 + + +def load_d(path: str) -> pd.DataFrame(): + return pd.read_csv(path, float_precision='round_trip', dtype='float64') + + +def plot_d(dat: pd.Series, fname: str = 'plot_input_data', colour: str = ''): + if colour == '': + colour = 'blue' + + pyplt.plot(dat, color=colour) + pyplt.xlim(0, len(dat)) + pyplt.ylim(min(dat) - 0.3, max(dat) + 0.3) + pyplt.xticks(size=plt_ticks_size()) + pyplt.yticks(size=plt_ticks_size()) + pyplt.savefig(fname + '.jpg') + # pplt.show(block=0) + # rework this + # ref: https://stackoverflow.com/a/46418284 + pyplt.close() + + +def mean(dat: pd.Series) -> float: + return dat.sum() / len(dat) + + +def variance(dat: pd.Series) -> float: + return sum(pow(dat - mean(dat), 2)) / len(dat) + + +def histogram(dat: pd.Series, bins: int = 10, fname: str = 'hist', colour: str = 'blue'): + pyplt.hist(dat, color=colour, bins=bins) + pyplt.xticks(size=plt_ticks_size()) + pyplt.yticks(size=plt_ticks_size()) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def distr_func(dat: pd.Series, fname: str = 'dist', colour: str = 'blue'): + dat.plot.density(color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def std_dev(dat: pd.Series) -> float: + return np.sqrt(variance(dat)) + + +def mean_diff(dat: pd.DataFrame) -> pd.Series: + return dat - dat.mean() + + +def covar(dat: pd.DataFrame) -> float: + dat = dat.apply(mean_diff) + return dat['u'].dot(dat['y']) / len(dat) + + +def covar_coeff(cov: float, std_dev_u: float, std_dev_y: float) -> float: + return cov / (std_dev_u * std_dev_y) + + +# m is the max permissible shift value. +def auto_corellation(dat: list, max_shift_n: int = .1) -> float: + v = [] + m = len(dat) * max_shift_n + cur_shift = 0 + + while m >= cur_shift: + r = 0 + + for i in range(len(dat) - cur_shift): + r += dat[i] * dat[i + cur_shift] + + r = r / len(dat) - cur_shift + v.append(r) + cur_shift += 1 + + return v + + +def mutual_corellation(dx: list, dy: list, max_shift_n: int = .1) -> float: + v = [] + m = len(dx) * max_shift_n + cur_shift = 0 + + while m >= cur_shift: + r = 0 + + for i in range(len(dx) - cur_shift): + r += dx[i] * dy[i + cur_shift] + + r = r / (len(dx) - cur_shift) + v.append(r) + cur_shift += 1 + + return v + + +def auto_covar(dat: list, max_shift_n: int = .1) -> float: + v = [] + mean = np.mean(dat) + m = len(dat) * max_shift_n + cur_shift = 0 + + while m >= cur_shift: + r = 0 + + for i in range(len(dat) - cur_shift): + r += (dat[i] - mean) * (dat[i + cur_shift] - mean) + + r = r / (len(dat) - cur_shift) + v.append(r) + cur_shift += 1 + + return v + + +def mutual_covar(dx: list, dy: list, max_shift_n: int = .1) -> float: + v = [] + mean_x, mean_y = np.mean(dx), np.mean(dy) + m = len(dx) * max_shift_n + cur_shift = 0 + + while m >= cur_shift: + r = 0 + + for i in range(len(dx) - cur_shift): + r += (dx[i] - mean_x) * (dy[i + cur_shift] - mean_y) + + r = r / (len(dx) - cur_shift) + v.append(r) + cur_shift += 1 + + return v + + +def plot_autocorellation(dat: pd.DataFrame, fname: str = 'autocorellation', colour: str = 'blue'): + d = auto_corellation(dat.tolist()) + pyplt.scatter(range(0, len(d)), d, color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def plot_mutual_corellation(d1: pd.DataFrame, d2: pd.DataFrame, fname: str = 'mutcorellation', colour: str = 'blue'): + d = mutual_corellation(d1.tolist(), d2.tolist()) + pyplt.scatter(range(0, len(d)), d, color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def plot_autocovariance(dat: pd.DataFrame, fname: str = 'autocovariance', colour: str = 'blue'): + d = auto_covar(dat.tolist()) + pyplt.scatter(range(0, len(d)), d, color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() + + +def plot_mutual_covariance(d1: pd.DataFrame, d2: pd.DataFrame, fname: str = 'mutcovariance', colour: str = 'blue'): + d = mutual_covar(d1.tolist(), d2.tolist()) + pyplt.scatter(range(0, len(d)), d, color=colour) + pyplt.savefig(fname + '.jpg') + pyplt.close() diff --git a/p1/p1/main.py b/p1/p1/main.py new file mode 100644 index 0000000..1e4ea04 --- /dev/null +++ b/p1/p1/main.py @@ -0,0 +1,72 @@ +import sys +import getopt +import numpy as np +import funcs as f + + +def main(argv): + inputfile = '' + try: + opts, args = getopt.getopt(argv, "hi:o:", ["ifile="]) + except getopt.GetoptError: + print('main.py -i ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print('main.py -i ') + sys.exit() + elif opt in ("-i", "--ifile"): + inputfile = arg + if inputfile != '': + print('Input file is', inputfile) + else: + print('no input file provided, see help (-h)\nexiting...') + sys.exit(1) + + # d = f.load_d("./dat.csv") + d = f.load_d(inputfile) + # d = d.astype('float64') + print(d.head()) + + # data plots + du = d['u'] + dy = d['y'] + f.plot_d(du, fname='u_input_plot') + f.plot_d(dy, fname='y_input_plot', colour='green') + + # mean and variance + mean_u = f.mean(d['u']) + mean_y = f.mean(d['y']) + + variance_u = f.variance(d['u']) + variance_y = f.variance(d['y']) + + hist_u = f.histogram(d['u'], fname='u_hist') + hist_y = f.histogram(d['y'], fname='y_hist', colour='green') + + dist_u = f.distr_func(d['u'], fname='u_dist') + dist_y = f.distr_func(d['y'], fname='y_dist', colour='green') + + cov = f.covar(d) + std_dev_u = f.std_dev(d['u']) + std_dev_y = f.std_dev(d['y']) + cov_c = f.covar_coeff(cov, std_dev_u, std_dev_y) + + print("data covariance:\n", d.cov()) + + # print the matrix. + print(np.array([[variance_u, cov], [cov, variance_y]])) + + f.plot_autocorellation(dat=d['u'], fname='u_autocorellation') + f.plot_autocorellation(dat=d['y'], fname='y_autocorellation', colour='green') + + f.plot_mutual_corellation(d1=d['u'], d2=d['y'], fname='mutual_corellation_uy') + + f.plot_autocovariance(dat=d['u'], fname='u_autocovariance') + f.plot_autocovariance(dat=d['y'], fname='y_autocovariance', colour='green') + + f.plot_mutual_covariance(d1=d['u'], d2=d['y'], fname='mutual_covariance_uy') + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/p1/poetry.lock b/p1/poetry.lock new file mode 100644 index 0000000..2ddd539 --- /dev/null +++ b/p1/poetry.lock @@ -0,0 +1,8 @@ +package = [] + +[metadata] +lock-version = "1.1" +python-versions = "^3.10" +content-hash = "53f2eabc9c26446fbcc00d348c47878e118afc2054778c3c803a0a8028af27d9" + +[metadata.files] diff --git a/p1/pyproject.toml b/p1/pyproject.toml new file mode 100644 index 0000000..c5deda5 --- /dev/null +++ b/p1/pyproject.toml @@ -0,0 +1,14 @@ +[tool.poetry] +name = "p1" +version = "0.0.0" +description = "" +authors = ["surtur "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.10" + + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/p1/tests/__init__.py b/p1/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/p1/tox.ini b/p1/tox.ini new file mode 100644 index 0000000..0d7b5db --- /dev/null +++ b/p1/tox.ini @@ -0,0 +1,6 @@ +[flake8] +indent_char='tab' +extend-ignore = W191, E501 +; ignore = +; # tabs are ok +; W191 diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..017cf3f --- /dev/null +++ b/shell.nix @@ -0,0 +1,16 @@ +( + import + ( + let + lock = builtins.fromJSON (builtins.readFile ./flake.lock); + in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { + src = ./.; + } +) +.shellNix