p2: implement file reading+basic data operations

also, update .pre-commit-config.yaml to:
* first change to ./p2, then run the tests (pre-commit deliberately does
  not allow this by design [1])
* for the above, the https://github.com/dnephin/pre-commit-golang repo
  had to be dropped as it does not allow any customisation either.

also, add some flake updates (would be better split into another commit,
I know):
* use baseurl var
* add statix to shell packages
* fix building p1

refs:
[1]: https://github.com/pre-commit/pre-commit/issues/1110

.pre-commit-config.yaml

@@ -31,15 +31,27 @@ repos:
         pass_filenames: false
         files: "flake.(nix|lock)$"
         language: system
-  - repo: https://github.com/dnephin/pre-commit-golang
-    rev: v0.5.0
-    hooks:
       - id: go-mod-tidy
+        name: go-mod-tidy
+        entry: bash -c 'cd ./p2 && go mod tidy'
+        files: '\.(go|mod|sum)$'
+        language: system
       - id: go-unit-tests
+        name: go-unit-tests
+        entry: bash -c "cd ./p2 && go test -v ./..."
+        files: '\.go$'
+        language: system
       - id: golangci-lint
+        name: golangci-lint
+        entry: bash -c 'cd p2 && golangci-lint run'
+        # types: [go]
+        pass_filenames: false
+        language: system
       - id: go-build
-  - repo: local
+        name: go-build
-    hooks:
+        entry: bash -c 'cd ./p2 && go build -v .'
+        language: system
+
       - id: nix-build-py
         name: nix build py
         entry: nix build .#p1

flake.nix

@@ -48,7 +48,7 @@
       pkgs = nixpkgsFor.${system};
       inherit (pkgs) lib overlays;
     in rec {
-      p1 = pkgs.${system}.poetry2nix.mkPoetryApplication {
+      p1 = nixpkgsFor.${system}.poetry2nix.mkPoetryApplication {
         name = "p1";
         projectDir = ./p1;
         meta = {
@@ -56,7 +56,7 @@
           homepage = baseurl + "p1";
           license = nixpkgs.lib.licenses.gpl3;
           maintainers = ["wanderer"];
-          platforms = nixpkgs.lib.platforms.linux ++ nixpkgs.lib.platforms.darwin;
+          platforms = lib.platforms.linux ++ lib.platforms.darwin;
         };
       };
 
@@ -86,7 +86,7 @@
           ];
 
           # dont't forget to update vendorSha256 whenever go.mod or go.sum change
-          vendorSha256 = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo=";
+          vendorSha256 = "sha256-Ns3ohAzZliK75fM6ryWubhfLBCVwU7CsZbuuzZrGaRY=";
 
           # In 'nix develop', we don't need a copy of the source tree
           # in the Nix store.
@@ -130,7 +130,7 @@
 
           meta = {
             description = "implementation of task 2 for ak9im";
-            homepage = "https://git.dotya.ml/wanderer/ak9im/p2";
+            homepage = baseurl + "p2";
             license = lib.licenses.gpl3;
             maintainers = ["wanderer"];
             platforms = lib.platforms.linux ++ lib.platforms.darwin;
@@ -189,6 +189,7 @@
           python3Packages.pynvim
           python3Packages.jedi
           pre-commit
+          statix
 
           # deps
           python3Packages.numpy

p2/README.md

@@ -1,3 +1,14 @@
 # p2
 
 this is a Go subproject containing code for task no. 2.
+
+### compile
+```sh
+go build -v .
+```
+
+### run
+```sh
+# pass a csv data file.
+./p2 -datafile=./data/m.csv
+```

p2/data.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"encoding/csv"
+	"errors"
+	"io"
+	"os"
+	"strconv"
+)
+
+func readData(f *os.File) ([][]float64, error) {
+	records := make([][]string, 0, 252)
+	r := csv.NewReader(f)
+
+	for {
+		record, err := r.Read()
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			return [][]float64{}, err
+		}
+
+		records = append(records, record)
+	}
+
+	data, err := parseRecords(records)
+	if err != nil {
+		return data, err
+	}
+
+	return data, nil
+}
+
+func parseRecords(r [][]string) ([][]float64, error) {
+	if len(r) == 0 {
+		return [][]float64{}, errors.New("ErrNoRecords")
+	}
+
+	// remove the header.
+	r = r[1:]
+
+	u := 0
+	y := 1
+	data := [][]float64{make([]float64, 0, len(r)), make([]float64, 0, len(r))}
+
+	for _, uy := range r {
+		fu, err := strconv.ParseFloat(uy[u], 64)
+		if err != nil {
+			return [][]float64{}, err
+		}
+
+		fy, err := strconv.ParseFloat(uy[y], 64)
+		if err != nil {
+			return [][]float64{}, err
+		}
+
+		data[u] = append(data[u], fu)
+		data[y] = append(data[y], fy)
+	}
+
+	return data, nil
+}

p2/data/m.csv

@@ -0,0 +1,252 @@
+u,y
+-0.562081627343819,0
+-0.165028051084386,-0.0848846176713865
+0.402381188889212,-0.00855074782595819
+0.982074771068094,0.0465851901457117
+0.263269436202603,0.169762201014319
+-0.666985599169035,0.054323197496775
+0.973284224035816,0.051746560207838
+0.827634883964264,-0.0173133818174058
+-0.9682645974533,0.0719123061293299
+-0.38735633780591,0.20323870152281
+0.0746079511356577,0.169900711685989
+0.0688996822894083,-0.00746558140650048
+-0.574496970313833,0.138126734045196
+0.875794598774889,-0.141852647292507
+-0.000517935026678185,-0.0988200155700192
+-0.182466612748088,0.00240103076106294
+-0.361500652209623,-0.0595347187662052
+-0.59349933061446,-0.0519022350351849
+-0.8156527401021,-0.106703366966349
+-0.0953996614997273,-0.246702573251392
+-0.0574753429076985,-0.0275445682385667
+-0.121339358911542,0.129163048642404
+0.0748514826758073,0.318297851793095
+0.431284669521863,0.223248747689702
+-0.869612254141649,0.251157772434315
+0.0878966139107462,0.212505320828667
+-0.120548874661582,0.107255808344614
+-0.998809915035409,0.0807653257237951
+0.672839759696666,0.0145081167947413
+-0.837878405041005,0.0253975644329584
+0.56773040516662,-2.39532856761661e-05
+-0.727090123913758,-0.10795814297631
+-0.796725244166667,-0.0909358736412822
+0.266859813251933,0.00995070727841241
+0.107822691606275,-0.0372011713223269
+0.626531425689595,-0.0651091197686131
+0.0361671741288934,-0.0487676892938064
+-0.224337562557467,-0.109582017325756
+0.457216727294594,-0.178916504141863
+0.322710993384342,0.0309704863456593
+0.658477414240352,0.0506431803910415
+0.388047331659145,0.0623101077769065
+0.503420185997812,0.0359911107419933
+-0.673837888833991,0.0193161516424325
+0.0880454252884004,-0.0383760770110465
+-0.439422153606742,-0.0359387768251584
+-0.558629704433787,-0.112251115812661
+0.03997697263955,-0.284971509798976
+-0.337650089216256,-0.148524557670779
+0.152345922380847,-0.140875672976474
+0.228251039622468,-0.0889231511951012
+-0.496944804441624,0.0465610558784346
+0.331006910340398,0.082068516189644
+-0.834986920391669,-0.151454048215957
+0.868060945471777,-0.160252554385469
+0.51354035433081,0.00985996775878407
+-0.532617003439282,-0.0174954084308043
+-0.0316390735244561,-0.128508236942345
+0.038349728583521,-0.0465041726496238
+-0.589317382122072,-0.037358615065556
+-0.67983367558561,-0.104981843201757
+0.153551720619924,-0.142242202275461
+0.0790093499603726,0.114229214547889
+-0.7900617140299,-0.0504704387615806
+-0.234356993452812,-0.0328588411720029
+-0.149617799161755,-0.137465346004397
+-0.305016663533178,-0.0635402938976893
+-0.909318567211422,0.0122852168966604
+0.399381979088943,-0.0946668447025222
+-0.127807520855129,-0.106796103666498
+0.18317640162221,-0.0990174921556541
+0.933819305120883,-0.0797645616353446
+0.381545237908859,0.135080185084484
+-0.31277324692941,0.173661209205918
+0.47434979000797,-0.0215205041095453
+-0.556687081026233,-0.0835518038778322
+0.179618671154426,-0.118914977199466
+0.0386732691147706,0.125386697025116
+-0.805134915656007,0.11315801596806
+-0.697566219464674,0.101667350424666
+-0.785638963703829,-0.106680765663954
+-0.0266376952764754,-0.0479903373971647
+-0.44670607030704,0.0353016950643207
+0.566004670954312,0.0364743295471811
+0.303029073077733,0.116905291426555
+0.404942760898239,0.0893797830464027
+-0.32298081057285,-0.0612423559178996
+0.72921899693516,-0.190651431253798
+-0.238783399219989,-0.201157561505088
+0.838983901701394,-0.153230865999878
+-0.445603677744793,0.0208606856497355
+0.498290091519379,0.131139698572563
+0.75811975717457,0.169855697665016
+0.75638357352297,-0.0257884500341728
+0.375155796937251,0.131076064227461
+0.257533313360779,-0.0528647895038988
+-0.032254330363243,0.0931748927364773
+-0.839355925954578,0.0348971935198497
+-0.321761685107724,-0.0442452312895603
+-0.141054466898113,-0.0925317420305877
+-0.251501206891379,0.0607248465670675
+-0.220999071011785,0.0882657343519309
+0.204498790765413,0.00799784484065389
+-0.751218585181617,-0.0836631271887629
+0.58976151775092,-0.150354940379272
+-0.0699859154736557,-0.0668967855012146
+-0.413913108135533,-0.135652120956737
+-0.18842738363353,-0.0721416993785926
+0.666755671457739,-0.0501845048503642
+-0.442408544683088,0.06883040791009
+-0.300941884192145,0.193375560177824
+-0.765524363967369,0.180540303154666
+0.859780197897824,0.153550465624583
+-0.6740838939669,0.139042359806811
+0.334695734705169,0.168269259962125
+0.710164911910037,0.0473114765318522
+-0.816398257304168,-0.0132560118645178
+-0.13099439587956,0.00642581834123225
+-0.745824782990769,0.0280274330024728
+0.146800391910039,0.0114525145825461
+-0.763205443398657,0.0268220879521105
+0.456613086842286,-0.0141582162933872
+0.438408563583348,-0.127737166963499
+0.342703012443475,-0.147812633292664
+0.698135564894478,-0.0137274640231074
+-0.670800844054111,0.000513004189454612
+0.585560483665001,-0.0957600030634544
+0.151705400623244,-0.126140672877399
+0.495937946017803,-0.0587775919654244
+0.828736296775162,-0.113602753385482
+-0.275617057120249,0.0404169153674105
+-0.129249441963271,0.0637441027533635
+-0.496089479651344,0.0520406223713919
+-0.747938023762749,-0.0434250282438795
+-0.837347029632585,-0.148086186883691
+-0.717641319948082,-0.139919043048457
+-0.936323192872258,-0.0605219822362255
+-0.554920240098108,0.0474048568421955
+0.719021106939307,-0.0947218176526316
+-0.947999400062486,0.0261778777337991
+0.171600238034316,-0.0677998584179937
+0.826004394249061,-0.123032216818122
+0.387852175807512,0.194905723500561
+0.0351910223416942,0.166027166275701
+0.479863301608648,0.101954502130607
+0.444585834371199,-0.00212100636994374
+-0.0155852180978214,0.0424901518175131
+0.692872747635875,-0.020136542529367
+-0.300434749247709,0.14017545466314
+0.495401680234541,0.0467252671274352
+0.1432539257888,0.0518152607705747
+0.323782088851455,0.00891091382416525
+-0.48723274072969,0.0656228532554857
+0.460745400498037,-0.0912258739936947
+-0.349475199053751,-0.137388220950933
+0.984357650384474,-0.0464662361149723
+0.717022333162382,-0.0290631543902982
+-0.0847671390905823,0.0842585339781461
+0.89878820343818,0.0387303686119073
+0.628449186509684,0.0326745671195254
+-0.6684710158354,0.100022791409108
+-0.353190103244591,-0.120791479513615
+0.883962914293615,-0.148576945580147
+-0.277615144512437,-0.184046870106486
+0.0948024723188963,-0.136855716266544
+-0.335375443722762,0.0311153392168198
+0.780905995415946,-0.0224553304744231
+0.0511205489985276,0.0788479869635113
+0.241345313955725,-0.0500833323004902
+0.851539127459535,-0.112609714052497
+-0.291347311479667,-0.132830197510762
+-0.843571841643924,-0.194825060471202
+-0.996671304105162,-0.218432230330827
+0.921911736913916,-0.159673404773765
+0.764604230301736,-0.199288865578464
+-0.532099923832389,0.116195428966659
+0.283330017367066,0.157707373781633
+0.258282310915311,0.140041359684938
+-0.935322373144013,0.196915951661411
+-0.830454336400355,0.120386701590254
+0.374635382264217,0.101826502163638
+0.821705620652859,0.209094203434554
+0.613403443067057,0.10826422633432
+-0.956468232887084,0.0569546117630172
+-0.186500359879108,0.172972783266203
+0.376801843464748,0.0341815581906813
+0.59207610860098,0.0638312199981932
+-0.037140366172949,-0.0440596218668656
+-0.434475232583692,0.0249241103881933
+-0.291389503186284,-0.037444191519866
+-0.0601084125508128,-0.10827784318051
+0.784298201922466,-0.233941733889692
+0.0130897644036867,-0.0292668767208536
+-0.471272126525302,-0.132352153493435
+-0.43183794684328,-0.0285887256597882
+-0.959193706493449,0.0195679991308332
+-0.913185525645123,0.145244999346624
+-0.414255243453316,0.0752718384166183
+0.363312282303028,0.076558551495326
+0.743653144568043,-0.0967207911101528
+-0.334610971312323,-0.057695839272927
+0.683896947504905,0.220457065560355
+0.381545025567312,0.249105557222865
+-0.480161357428954,0.218030510334781
+0.304930325273858,0.0644631852618431
+0.562712400948029,-0.0138504593059087
+-0.498690735315294,0.0493680673636072
+0.917412161788629,0.0829444150386858
+-0.0910431757061944,0.212697184586624
+-0.639183673839636,0.230104278405291
+-0.139293632069274,0.159652227148624
+0.722858748269667,0.0954345880223326
+-0.576794941712541,0.207757724506451
+0.931673358162713,-0.0422537892114052
+0.335037476073502,0.121191599709049
+-0.526334091800421,0.135758709437636
+-0.852575754678145,0.0466947927713023
+-0.609541804347905,0.125927079414675
+0.671690425682669,-0.0078502938544791
+0.0763494088670003,-0.0585132867888244
+0.0892805201417211,-0.175864475624874
+-0.323095335309904,-0.154157927887642
+-0.174369967158125,0.0522737524138117
+-0.524507459962977,0.0783792135121095
+0.957202981206217,0.0897387931304263
+0.813413716765779,0.0596736907760852
+0.283293277622803,0.0372312933723317
+0.513900782686612,-0.150468366623803
+0.395650216096849,-0.147955284625277
+-0.0823185420978435,-0.0223826480219779
+0.1341197896442,-0.0280534253211324
+0.0899726520711428,0.0513034519514842
+0.817904155616604,8.59935647047512e-05
+-0.656541830700143,-0.0351927114589313
+-0.151372477017051,-0.0139098899105425
+0.475657624879692,-0.0687908511025121
+-0.941745635094934,-0.0620316597861909
+0.0241828938127415,-0.0424930929299225
+-0.0225139358185762,0.151388215635737
+-0.100346996961323,0.154763082869366
+0.193473809954465,0.0271937744025612
+0.640271583404519,0.0105602365003001
+0.59476472232247,0.0572940959449519
+0.30324439671973,-0.0612133530446115
+-0.993561718609911,0.0921513064661191
+-0.246076793058811,-0.000518372223260732
+0.923166690358504,0.0650959749032007
+-0.856037346579152,0.0681306364260466
+-0.453166298313609,-0.103159953315381
+0.572474384481308,-0.116014111229541
+-0.435653748659256,-0.0938795633662015

p2/go.mod

@@ -1,3 +1,5 @@
 module git.dotya.ml/wanderer/ak9im/p2
 
 go 1.19
+
+require gonum.org/v1/gonum v0.12.0

p2/go.sum

@@ -0,0 +1,3 @@
+golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3 h1:n9HxLrNxWWtEb1cA950nuEEj3QnKbtsCJ6KjcgisNUs=
+gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o=
+gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY=

p2/main.go

@@ -1,3 +1,10 @@
 package main
 
-func main() {}
+import "log"
+
+func main() {
+	err := run()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

p2/run.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+
+	"git.dotya.ml/wanderer/ak9im/p2/stats"
+)
+
+var datafile = flag.String("datafile", "", "read data from this file")
+
+func run() error {
+	flag.Parse()
+
+	if *datafile != "" {
+		f, err := os.Open(*datafile)
+		if err != nil {
+			return err
+		}
+
+		defer f.Close()
+
+		u := 0
+		y := 1
+
+		data, err := readData(f)
+		if err != nil {
+			return err
+		}
+
+		meanU := stats.Mean(data[u])
+		meanY := stats.Mean(data[y])
+		varianceU := stats.Variance(data[u])
+		varianceY := stats.Variance(data[y])
+
+		log.Printf("len(data): %d\n", len(data[u]))
+
+		log.Printf("means - u: %v, y: %v", meanU, meanY)
+
+		log.Printf("variance - u: %v, y: %v", varianceU, varianceY)
+	}
+
+	return nil
+}

p2/stats/stats.go

@@ -0,0 +1,64 @@
+package stats
+
+import (
+	"math"
+
+	"gonum.org/v1/gonum/stat"
+)
+
+// Mean returns a mean value for f []float64.
+func Mean(f []float64) float64 {
+	return stat.Mean(f, nil)
+}
+
+// Variance calculates the variance of f []float64.
+func Variance(f []float64) float64 {
+	return stat.Variance(f, nil)
+}
+
+func Autocorrelate(f []float64, maxShift float64) []float64 {
+	fLen := len(f)
+	m := float64(fLen) * maxShift
+	v := make([]float64, int(m), int(m)*4)
+	currentShift := 0
+
+	for m >= float64(currentShift) {
+		var r float64
+
+		for i := 0; i < fLen-currentShift; i++ {
+			r += f[i] * f[i+currentShift]
+		}
+
+		// r = r * math.Pow(float64(fLen)-float64(currentShift), -1)
+		r *= float64(1 / (fLen - currentShift))
+
+		v = append(v, r)
+
+		currentShift++
+	}
+
+	return v
+}
+
+func AutocorrelateMP(f []float64, maxShift float64) []float64 {
+	fLen := len(f)
+	m := float64(fLen) * maxShift
+	v := make([]float64, int(m), 10)
+	currentShift := 0
+
+	for m >= float64(currentShift) {
+		var r float64
+
+		for i := 0; i < fLen-currentShift; i++ {
+			r += f[i] * f[i+currentShift]
+		}
+
+		r *= math.Pow(float64(fLen)-float64(currentShift), -1)
+
+		v = append(v, r)
+
+		currentShift++
+	}
+
+	return v
+}

p2/stats/stats_test.go

@@ -0,0 +1,60 @@
+package stats
+
+import "testing"
+
+func TestAutocorrelate(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		maxShift float64
+		in       []float64
+		out      []float64
+	}{
+		{
+			desc:     "autocorrelate a 10-long slice",
+			maxShift: .1,
+			in:       []float64{.4, .5, .23, 2.5, -3.6, .4, -0.12, 2.2, 0.02, 14.4},
+			out:      []float64{0, 9.529999999999998},
+		},
+		{
+			desc:     "autocorrelate a 20-long slice",
+			maxShift: .1,
+			in:       []float64{.4, .5, .23, 2.5, -3.6, .4, -0.12, 2.2, 0.02, 14.4, .4, .5, .23, 2.5, -3.6, .4, -0.12, 2.2, 0.02, 14.4},
+			out:      []float64{0, 13.299999999999994, -152.43040000000002},
+		},
+	}
+
+	for _, tC := range testCases {
+		t.Run(tC.desc, func(t *testing.T) {
+			want := tC.out
+			got := Autocorrelate(tC.in, tC.maxShift)
+
+			if len(want) != len(got) {
+				t.Errorf("Slices are of different sizes, want: %d, got: %d",
+					len(want), len(got),
+				)
+			}
+
+			for i := range want {
+				if want[i] != got[i] {
+					t.Errorf("Unexpected output for %q: expected %+v, actual %+v",
+						tC.desc, want, got,
+					)
+				}
+			}
+		})
+	}
+}
+
+func BenchmarkAutocorrelate(b *testing.B) {
+	// run the function b.N times.
+	for n := 0; n < b.N; n++ {
+		Autocorrelate([]float64{.4, .5, .23, 2.5, -3.6, .4, -0.12, 2.2, 0.02, 14.4}, .1)
+	}
+}
+
+func BenchmarkAutocorrelateMP(b *testing.B) {
+	// run the function b.N times.
+	for n := 0; n < b.N; n++ {
+		AutocorrelateMP([]float64{.4, .5, .23, 2.5, -3.6, .4, -0.12, 2.2, 0.02, 14.4}, .1)
+	}
+}