flake,go,pre-commit: add p3

.pre-commit-config.yaml

@@ -54,11 +54,20 @@ repos:
         # python files.
         files: '\.*.(nix|lock|py)$'
         language: system
-      - id: nix-build-go
+      - id: nix-build-go-2
-        name: nix build go
+        name: nix build go p2
         entry: nix build .#p2
         pass_filenames: false
         # trigger this hook on changes to any of nix (also flake.lock) files
         # and go's mod or sum files
-        files: '\.*.(nix|lock|mod|sum)$'
+        # files: '\.*.(nix|lock|mod|sum)$'
+        files: '\.*.(nix|lock|)|./p2/*.(mod|sum)$'
+        language: system
+      - id: nix-build-go-p3
+        name: nix build go p3
+        entry: nix build .#p3
+        pass_filenames: false
+        # trigger this hook on changes to any of nix (also flake.lock) files
+        # and go's mod or sum files
+        files: '\.*.(nix|lock|)|./p3/*.(mod|sum)$'
         language: system

flake.nix

@@ -144,6 +144,90 @@
           };
         };
 
+      p3 = with pkgs;
+        buildGo119Module rec {
+          pname = "p3";
+          buildInputs = [
+            go_1_19
+            gcc
+
+            python3Packages.numpy
+            python3Packages.pandas
+            python3Packages.matplotlib
+            python3Packages.scipy
+            python3Packages.pybindgen
+            libxcrypt
+          ];
+          nativeBuildInputs = [pkgconfig];
+
+          overrideModAttrs = _: {
+            # GOPROXY = "direct";
+            GOFLAGS = "-buildmode=pie -trimpath -mod=readonly -modcacherw";
+          };
+
+          inherit version;
+          doCheck = false;
+          # use go.mod for managing go deps, instead of vendor-only dir
+          proxyVendor = true;
+          tags = []; # go "-tags" to build with
+          ldflags = [
+            "-s"
+            "-w"
+            "-X main.version=${version}"
+          ];
+
+          # dont't forget to update vendorSha256 whenever go.mod or go.sum change
+          vendorSha256 = "sha256-QXY1MN32g4WSyXfZDpR2C5TzP0lHgmecT7Jv4Dtf1Bk=";
+
+          # In 'nix develop', we don't need a copy of the source tree
+          # in the Nix store.
+          src = nix-filter.lib.filter {
+            # when in doubt, check out
+            # https://github.com/numtide/nix-filter#design-notes
+            # tl;dr: it'd be best to include folders, however there are
+            # currently issues with that approach.
+            root = lib.cleanSource ./p3;
+            exclude = [
+              ./p3/README.md
+
+              ./p3
+
+              ./flake.nix
+              ./flake.lock
+              ./default.nix
+              ./shell.nix
+              ./overlay.nix
+
+              ./README.md
+
+              ./.envrc
+              ./.gitattributes
+              ./.gitignore
+              ./.golangci.yml
+              ./.editorconfig
+              ./.pre-commit-config.yaml
+
+              # program output
+              ./out
+              ./res
+
+              # nix result symlink
+              ./result
+
+              # the entire .git folder
+              ./.git
+            ];
+          };
+
+          meta = {
+            description = "implementation of task 3 for ak9im";
+            homepage = baseurl + "p3";
+            license = lib.licenses.gpl3;
+            maintainers = ["wanderer"];
+            platforms = lib.platforms.linux ++ lib.platforms.darwin;
+          };
+        };
+
       default = p1;
     });
 
@@ -156,6 +240,10 @@
         type = "app";
         program = "${self.packages.${system}.${projname}}/bin/p2";
       };
+      p3 = {
+        type = "app";
+        program = "${self.packages.${system}.${projname}}/bin/p3";
+      };
       default = p1;
     });
 

p3/README.md

@@ -0,0 +1,29 @@
+# p3
+
+this is a Go subproject containing code for task no. 3.
+Python is used for visualisation.
+
+### compile
+```sh
+go build -v .
+```
+
+### run
+to compute correlations, impulse function:
+```sh
+# pass a csv data file.
+./p2 -datafile=./data/m.csv
+```
+
+to visualise the computed data:
+```sh
+python visualise.py
+```
+
+alternatively, from current folder you could do both of the above in a single
+step:
+```diff
+# compute stuff and visualise.
+-./p2 -datafile=./data/m.csv
++./p2 -datafile=./data/m.csv -vis
+```

p3/data.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"encoding/csv"
+	"errors"
+	"io"
+	"os"
+	"strconv"
+)
+
+func readData(f *os.File) ([][]float64, error) {
+	records := make([][]string, 0, 252)
+	r := csv.NewReader(f)
+
+	for {
+		record, err := r.Read()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return [][]float64{}, err
+		}
+
+		records = append(records, record)
+	}
+
+	data, err := parseRecords(records)
+	if err != nil {
+		return data, err
+	}
+
+	return data, nil
+}
+
+func parseRecords(r [][]string) ([][]float64, error) {
+	if len(r) == 0 {
+		return [][]float64{}, errors.New("ErrNoRecords")
+	}
+
+	// remove the header.
+	r = r[1:]
+
+	u := 0
+	y := 1
+	data := [][]float64{make([]float64, 0, len(r)), make([]float64, 0, len(r))}
+
+	for _, uy := range r {
+		fu, err := strconv.ParseFloat(uy[u], 64)
+		if err != nil {
+			return [][]float64{}, err
+		}
+
+		fy, err := strconv.ParseFloat(uy[y], 64)
+		if err != nil {
+			return [][]float64{}, err
+		}
+
+		data[u] = append(data[u], fu)
+		data[y] = append(data[y], fy)
+	}
+
+	return data, nil
+}
+
+func readFile(s *string) ([][]float64, error) {
+	f, err := os.Open(*s)
+	if err != nil {
+		return [][]float64{}, err
+	}
+
+	defer f.Close()
+
+	data, err := readData(f)
+	if err != nil {
+		return [][]float64{}, err
+	}
+
+	return data, nil
+}
+
+func saveStuff(
+	meanU, meanY,
+	varianceU, varianceY,
+	cov float64,
+	autocorrelationU, autocorrelationY,
+	mutCorrelationUY, mutCorrelationYU []float64,
+) error {
+	fFnames := map[string]float64{
+		"mean_u":        meanU,
+		"mean_y":        meanY,
+		"variance_u":    varianceU,
+		"variance_y":    varianceY,
+		"covariance_uy": cov,
+	}
+	sFnames := map[string][]float64{
+		"autocorrelation_u":     autocorrelationU,
+		"autocorrelation_y":     autocorrelationY,
+		"mutual_correlation_uy": mutCorrelationUY,
+		"mutual_correlation_yu": mutCorrelationYU,
+	}
+	prefix := "data/"
+	suffix := ".txt"
+
+	for k, v := range fFnames {
+		f, err := os.Create(prefix + k + suffix)
+		if err != nil {
+			return err
+		}
+
+		defer f.Close()
+
+		_, err = f.WriteString(strconv.FormatFloat(v, 'f', 64, 64))
+		if err != nil {
+			return err
+		}
+	}
+
+	suffix = ".csv"
+
+	for k, v := range sFnames {
+		f, err := os.Create(prefix + k + suffix)
+		if err != nil {
+			return err
+		}
+
+		defer f.Close()
+
+		w := csv.NewWriter(f)
+
+		s := recordsFromFloat64Slice(v)
+
+		err = w.WriteAll(s)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func recordsFromFloat64Slice(f []float64) [][]string {
+	s := make([][]string, 0, len(f))
+
+	for i := range f {
+		tmp := strconv.FormatFloat(f[i], 'f', 64, 64)
+
+		s = append(s, []string{tmp})
+	}
+
+	return s
+}

p3/go.mod

@@ -0,0 +1,7 @@
+module git.dotya.ml/wanderer/ak9im/p3
+
+go 1.19
+
+require git.dotya.ml/wanderer/ak9im/p2 v0.0.0-20230227032746-9910d7702660
+
+require gonum.org/v1/gonum v0.12.0 // indirect

p3/go.sum

@@ -0,0 +1,5 @@
+git.dotya.ml/wanderer/ak9im/p2 v0.0.0-20230227032746-9910d7702660 h1:Zy8MLNXG6OyLiQnvcD1YdsJbDXpSpL4FHm8s8FAmWSY=
+git.dotya.ml/wanderer/ak9im/p2 v0.0.0-20230227032746-9910d7702660/go.mod h1:YniKHLVg0WXF8jNgIa4jIWvGicLLX0zWOilyIPA3EjU=
+golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 h1:n9HxLrNxWWtEb1cA950nuEEj3QnKbtsCJ6KjcgisNUs=
+gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o=
+gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY=

p3/main.go

@@ -0,0 +1,10 @@
+package main
+
+import "log"
+
+func main() {
+	err := run()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

p3/run.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os/exec"
+
+	"git.dotya.ml/wanderer/ak9im/p2/stats"
+)
+
+var (
+	datafile = flag.String("datafile", "", "read data from this file")
+	vis      = flag.Bool("vis", false, "run 'python visualise.py' to produce visualisations")
+)
+
+func run() error {
+	flag.Parse()
+
+	if *datafile != "" {
+		data, err := readFile(datafile)
+		if err != nil {
+			return err
+		}
+
+		u := 0
+		y := 1
+
+		meanU := stats.Mean(data[u])
+		meanY := stats.Mean(data[y])
+		varianceU := stats.Variance(data[u])
+		varianceY := stats.Variance(data[y])
+
+		maxShift := 0.1
+		autocorrelationU := stats.Autocorrelate(data[u], maxShift)
+		autocorrelationY := stats.Autocorrelate(data[y], maxShift)
+
+		mutCorrelationUY, err := stats.MutCorrelate(data[u], data[y], maxShift)
+		if err != nil {
+			return err
+		}
+
+		mutCorrelationYU, err := stats.MutCorrelate(data[y], data[u], maxShift)
+		if err != nil {
+			return err
+		}
+
+		cov := stats.Covariance(data[u], data[y])
+
+		log.Printf("len(data): %d", len(data[u]))
+
+		log.Printf("means - u: %v, y: %v", meanU, meanY)
+
+		log.Printf("variance - u: %v, y: %v", varianceU, varianceY)
+
+		log.Printf("len(autocorrelationU): %d", len(autocorrelationU))
+		log.Printf("autocorrelationU: %v", autocorrelationU)
+		log.Printf("autocorrelationY: %v", autocorrelationY)
+		log.Printf("mutual correlation U,Y: %v", mutCorrelationUY)
+		log.Printf("mutual correlation Y,U: %v", mutCorrelationYU)
+		log.Printf("covariance U,Y: %v", cov)
+		log.Printf("correlation U,Y: %v", stats.Correlation(data[u], data[y]))
+
+		err = saveStuff(
+			meanU, meanY,
+			varianceU, varianceY,
+			cov,
+			autocorrelationU, autocorrelationY,
+			mutCorrelationUY, mutCorrelationYU,
+		)
+		if err != nil {
+			return err
+		}
+	}
+
+	if *vis {
+		err := visualise()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func visualise() error {
+	red := "\033[31m"
+	cyan := "\033[36m"
+	reset := "\033[0m"
+	f := "visualise.py"
+
+	log.Printf("running %s`python %s`%s", cyan, f, reset)
+
+	out, err := exec.Command("python", f).CombinedOutput()
+	if err != nil {
+		log.Printf("visualise failed with:\n\n%s%s%s\n", red, out, reset)
+		return err
+	}
+
+	return nil
+}

p3/visualise.py

@@ -0,0 +1 @@
+print("visualise")