<|start_filename|>cmd/huggingfaceimporter/internal/search.go<|end_filename|>
// Copyright 2020 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package internal

import (
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// ModelCard is a model in the search results.
type ModelCard struct {
	// LastModified is the last time this model was modified.
	LastModified time.Time `json:"lastModified"`
	// ModelID is the model identifier
	ModelID string `json:"id"`
	// Author is the author's name.
	Author string `json:"author"`
	// Downloads is the number of downloads.
	Downloads int `json:"downloads"`
	// Tags is the list of tags.
	Tags []string `json:"tags"`
}

// LookupFromHuggingFace looks up a search query, empty query returns all.
func LookupFromHuggingFace(searchQuery string) (string, error) {
	loc, err := url.Parse("https://huggingface.co/models")
	if err != nil {
		return "", err
	}
	if searchQuery != "" {
		loc.RawQuery = url.Values{
			"search": []string{searchQuery},
		}.Encode()
	}

	resp, err := http.Get(loc.String())
	if err != nil {
		return "", err
	}

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}

	dataStr := string(data)
	startSubstr := "window.getAllModels = () => ["
	idx := strings.Index(dataStr, startSubstr)
	if idx < 0 {
		return "", errors.New("getAllModels not found")
	}
	idx += len(startSubstr) - 1
	dataStr = dataStr[idx:]
	endSubstr := "}\n];"
	endIdx := strings.Index(dataStr, endSubstr)
	dataStr = dataStr[:endIdx+len(endSubstr)-1]
	return dataStr, nil
}

// ParseSearchResults parses search results json.
func ParseSearchResults(dataJSON []byte) ([]*ModelCard, error) {
	var res []*ModelCard
	if err := json.Unmarshal(dataJSON, &res); err != nil {
		return nil, err
	}
	return res, nil
}


<|start_filename|>pkg/nlp/tokenizers/sentencepiece/internal/sentencepiece/sentencepiece_proto.go<|end_filename|>
// Copyright (c) 2020 <NAME>. All rights reserved.
// This work is licensed under the terms of the MIT license.
// For a copy, see <https://opensource.org/licenses/MIT>.

package sentencepiece

import (
	"fmt"
	"os"

	"google.golang.org/protobuf/proto"
)

// NewSentencepieceFromFile creates sentencepiece from file.
func NewSentencepieceFromFile(filename string, lowercase bool) (Sentencepiece, error) {
	s := NewEmptySentencepiece(lowercase)
	bytes, err := os.ReadFile(filename)
	if err != nil {
		return s, fmt.Errorf("unable to read file : %s, err %v", filename, err)
	}
	var model ModelProto
	err = proto.Unmarshal(bytes, &model)
	if err != nil {
		return s, fmt.Errorf("unable to read model file : %s, err %v", filename, err)
	}

	count := 0
	for i, piece := range model.GetPieces() {
		typ := piece.GetType()
		word := piece.GetPiece()
		switch typ {
		case ModelProto_SentencePiece_NORMAL, ModelProto_SentencePiece_USER_DEFINED:
			s.insert(word, piece.GetScore(), int32(i))
		case ModelProto_SentencePiece_UNKNOWN:
			s.SetUnknownIndex(int32(i))
		case ModelProto_SentencePiece_CONTROL:
			s.SetControlWord(word, int32(i))
		}
		count++
	}

	return s, nil
}


<|start_filename|>pkg/utils/io_test.go<|end_filename|>
// Copyright 2019 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package utils

import (
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"os"
	"testing"
)

func TestFileSerialization(t *testing.T) {
	tempFile, err := os.CreateTemp("", "spago-serialization-test-")
	require.Nil(t, err)
	fileName := tempFile.Name()
	defer os.Remove(fileName)

	type MyStruct struct {
		X int
		S *MyStruct
	}

	obj1 := MyStruct{
		X: 1,
		S: &MyStruct{X: 2},
	}

	err = SerializeToFile(fileName, obj1)
	require.Nil(t, err)

	var obj2 MyStruct
	err = DeserializeFromFile(fileName, &obj2)
	require.Nil(t, err)
	assert.Equal(t, obj1, obj2)
}

func TestCountLines(t *testing.T) {
	tempFile, err := os.CreateTemp("", "spago-count-lines-test-")
	require.Nil(t, err)
	fileName := tempFile.Name()
	defer os.Remove(fileName)

	err = os.WriteFile(fileName, []byte("foo\nbar\nbaz\n"), 0644)
	require.Nil(t, err)

	n, err := CountLines(fileName)
	require.Nil(t, err)
	assert.Equal(t, 3, n)
}


<|start_filename|>pkg/nlp/gbst/model.go<|end_filename|>
// Copyright 2021 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gbst provides an implementation of the GBST (gradient-based subword tokenization)
// module from the Charformer paper (https://arxiv.org/abs/2106.12672).
// It automatically learns latent sub-words representations from characters in a data-driven fashion.
package gbst

import (
	"encoding/gob"
	"math"

	mat "github.com/nlpodyssey/spago/pkg/mat32"
	"github.com/nlpodyssey/spago/pkg/ml/ag"
	"github.com/nlpodyssey/spago/pkg/ml/nn"
	"github.com/nlpodyssey/spago/pkg/ml/nn/convolution1d"
	"github.com/nlpodyssey/spago/pkg/ml/nn/linear"
)

var (
	_ nn.Model = &Model{}
)

// Model contains the serializable parameters.
type Model struct {
	nn.BaseModel
	Config
	Conv     []*convolution1d.Model
	Proj     []*convolution1d.Model
	Scorer   *linear.Model
	PadUntil int
}

// Config provides configuration settings for GBST (Gradient Based Subword Tokenization)
type Config struct {
	InputSize               int
	MaxBlockSize            int
	BlockSize               []int
	DownsampleFactor        int
	ScoreConsensusAttention bool
}

func init() {
	gob.Register(&Model{})
}

// New returns a new convolution Model, initialized according to the given configuration.
func New(config Config) *Model {
	if config.DownsampleFactor > config.MaxBlockSize {
		panic("gbst: downsample factor must be lower than maxiumum block size")
	}
	return &Model{
		Config: config,
		Conv: makeConvModels(config.InputSize, convolution1d.Config{
			KernelSizeX:    1,
			KernelSizeY:    config.MaxBlockSize,
			YStride:        1,
			InputChannels:  1,
			OutputChannels: 1,
			Mask:           nil,
			DepthWise:      false,
			Activation:     ag.OpIdentity,
		}),
		Proj: makeConvModels(config.InputSize, convolution1d.Config{
			KernelSizeX:    config.InputSize,
			KernelSizeY:    1,
			YStride:        1,
			InputChannels:  1,
			OutputChannels: 1,
			Mask:           nil,
			DepthWise:      false,
			Activation:     ag.OpIdentity,
		}),
		Scorer:   linear.New(config.InputSize, 1),
		PadUntil: lcm(config.BlockSize[0], config.BlockSize[1], config.BlockSize[2:len(config.BlockSize)]...),
	}
}

func makeConvModels(n int, config convolution1d.Config) []*convolution1d.Model {
	ms := make([]*convolution1d.Model, n)
	for i := 0; i < n; i++ {
		ms[i] = convolution1d.New(config)
	}
	return ms
}

// Forward performs the forward step for each input node and returns the result.
func (m *Model) Forward(xs ...ag.Node) []ag.Node {
	l := len(xs)
	ys := make([]ag.Node, l)
	xs = m.padToMultiple(xs...)
	stackedIn := m.Graph().Stack(xs...)
	transposedStackedIn := m.Graph().T(stackedIn)
	stackedConvolvedXs := m.convolution(transposedStackedIn)
	convolvedEmbeddings := m.projection(stackedConvolvedXs, transposedStackedIn.Value().Rows())
	meanSequences := m.blocksMean(convolvedEmbeddings, l)
	scores := m.scorer(meanSequences, l)
	ys = m.weightSequence(meanSequences, scores, l)
	return m.downsample(ys)
}

// padToMultiple pads the sequence until lcm of blocks length
func (m *Model) padToMultiple(xs ...ag.Node) []ag.Node {
	n := nextDivisibleLength(len(xs), m.PadUntil)
	for i := 0; i < n; i++ {
		if i >= len(xs) {
			xs = append(xs, m.Graph().NewVariable(mat.NewEmptyVecDense(m.InputSize), false))
		}
	}
	return xs
}

// blockMean applies the average pooling of different size blocks. For example, considering blocks size 2
// out[0] = out[1] = average(in[0], in[1]);   out[2] = out[3] = average(in[2], in[3]) and so on.
func (m *Model) blockMean(sequence []ag.Node, outputSize, blockSize int) []ag.Node {
	g := m.Graph()
	out := make([]ag.Node, outputSize)
	l := len(sequence)
	for i := 0; i < l; i++ {
		if i < outputSize {
			if i%blockSize == 0 {
				if i+blockSize <= l {
					out[i] = g.Mean(sequence[i : i+blockSize])
				} else {
					out[i] = g.Mean(sequence[i:l])
				}
			} else {
				out[i] = g.Identity(out[i-1])
			}
		}
	}
	if l < outputSize {
		for k := l; k < outputSize; k++ {
			out[k] = g.Identity(out[l-1])
		}
	}

	return out
}

// seqMean applies the average pooling of the output sequence.
// This method, in contrast to the blockMean, returns a downsampled sequence.
// For example, considering blocks size 2
// out[0] =  average(in[0], in[1]);   out[1] = average(in[2], in[3]) and so on.
func (m *Model) seqMean(sequence []ag.Node, outputSize, blockSize int) []ag.Node {
	g := m.Graph()
	out := make([]ag.Node, outputSize)
	l := len(sequence)
	j := 0
	for i := 0; i < l; i++ {
		if i%blockSize == 0 {
			if i+blockSize <= l {
				out[j] = g.Mean(sequence[i : i+blockSize])
				j++
			} else {
				out[j] = g.Mean(sequence[i:l])
			}
		}
	}
	return out
}

// scorer is a parametrized linear function that produce a score for each candidate block.
func (m *Model) scorer(blocksSequence [][]ag.Node, length int) []ag.Node {
	g := m.Graph()
	scores := make([]ag.Node, length)
	for i := 0; i < length; i++ {
		ff := make([]ag.Node, len(m.BlockSize))
		for j, seq := range blocksSequence {
			ff[j] = m.Scorer.Forward(g.T(seq[i]))[0]
		}
		scores[i] = g.Softmax(g.Concat(ff...))
	}
	if m.Config.ScoreConsensusAttention {
		return m.scoreWithConsensusAttention(scores, length)
	}
	return scores
}

func (m *Model) scoreWithConsensusAttention(scores []ag.Node, length int) []ag.Node {
	g := m.Graph()
	scoresAttention := make([]ag.Node, length)
	stackedScores := g.Stack(scores...)
	dotProd := g.Mul(stackedScores, g.T(stackedScores))
	for i := 0; i < length; i++ {
		row := g.RowView(dotProd, i)
		softmaxAttention := g.Softmax(row)
		scoresAttention[i] = g.T(g.Mul(g.T(softmaxAttention), stackedScores))
	}
	return scoresAttention
}

// weightSequence calculates the weighted sum between all blocks and their score.
func (m *Model) weightSequence(blocksSequence [][]ag.Node, scores []ag.Node, length int) []ag.Node {
	g := m.Graph()
	out := make([]ag.Node, length)
	for i := 0; i < length; i++ {
		sepScores := nn.SeparateVec(m.Graph(), scores[i])
		weightedScore := m.Graph().NewVariable(mat.NewEmptyVecDense(m.InputSize), true)
		for j, seq := range blocksSequence {
			weightedScore = g.Add(weightedScore, g.ProdScalar(seq[i], sepScores[j]))
		}
		out[i] = weightedScore
	}
	return out
}

// convolution applies 1d convolution through the sequence, for each character dimension.
func (m *Model) convolution(xs ag.Node) ag.Node {
	convolved := make([]ag.Node, xs.Value().Rows())
	for i := 0; i < xs.Value().Rows(); i++ {
		row := m.Graph().RowView(xs, i)
		convolved[i] = m.Conv[i].Forward(row)[0]
	}
	return m.Graph().Stack(convolved...)
}

// projection applies a projection after the convolution through the sequence, for each character dimension.
func (m *Model) projection(inStackedVectors ag.Node, length int) []ag.Node {
	projectedXs := make([]ag.Node, length)
	for i := 0; i < length; i++ {
		projectedXs[i] = m.Proj[i].Forward(inStackedVectors)[0]
	}
	stackedProjectedXs := m.Graph().Stack(projectedXs...)
	stackedProjectedXs = m.Graph().T(stackedProjectedXs)
	convolvedEmbeddings := make([]ag.Node, stackedProjectedXs.Value().Rows())
	for i := range convolvedEmbeddings {
		convolvedEmbeddings[i] = m.Graph().RowView(stackedProjectedXs, i)
	}
	return convolvedEmbeddings
}

// blocksMean calculates the average pooling for each block of length 1 .. m.Blocksize, for the sequence of length l
func (m *Model) blocksMean(convolvedEmbeddings []ag.Node, length int) [][]ag.Node {
	meanSequences := make([][]ag.Node, len(m.BlockSize))
	meanSequences[0] = make([]ag.Node, length)
	maxLen := len(convolvedEmbeddings)
	for i := 0; i < length; i++ {
		if i < maxLen {
			meanSequences[0][i] = m.Graph().Identity(convolvedEmbeddings[i])
		} else {
			meanSequences[0][i] = m.Graph().Identity(convolvedEmbeddings[maxLen-1])
		}
	}
	for i := 1; i < len(meanSequences); i++ {
		meanSequences[i] = m.blockMean(convolvedEmbeddings, length, m.BlockSize[i])
	}
	return meanSequences
}

// downsample reduces the sequence by the downsample factor
func (m *Model) downsample(xs []ag.Node) []ag.Node {
	if m.DownsampleFactor < 2 {
		return xs
	}
	return m.seqMean(xs, int(math.Ceil(float64(len(xs))/float64(m.DownsampleFactor))), m.DownsampleFactor)
}


<|start_filename|>pkg/nlp/gbst/model_test.go<|end_filename|>
// Copyright 2021 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gbst

import (
	mat "github.com/nlpodyssey/spago/pkg/mat32"
	"github.com/nlpodyssey/spago/pkg/ml/ag"
	"github.com/nlpodyssey/spago/pkg/ml/nn"
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestModel_Forward(t *testing.T) {
	model := newTestModel()
	g := ag.NewGraph()
	proc := nn.ReifyForTraining(model, g).(*Model)

	// == Forward
	sequence := []ag.Node{
		g.NewVariable(mat.NewVecDense([]mat.Float{0.5234, 0.8113}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-1.7743, -0.5153}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{0.9396, -1.6837}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.2936, 0.3536}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.5461, 0.6564}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.7707, -1.346}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{1.8908, 0.2333}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.1716, -0.8175}), true),
	}

	y := proc.Forward(sequence...)
	assert.InDeltaSlice(t, []mat.Float{0.40308161, -0.0114564}, y[0].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.56553651, 0.210189423}, y[1].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.23214475, -0.20046316}, y[2].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.29265471, -0.17325710}, y[3].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.61478526, 0.207365217}, y[4].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.57548401, 0.177836613}, y[5].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.22608360, -0.202010417}, y[6].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.33525165, -0.079017393}, y[7].Value().Data(), 1.0e-05)
}

func TestModel_ForwardScoreConsensusAttention(t *testing.T) {
	model := newTestModel()
	model.Config.ScoreConsensusAttention = true
	g := ag.NewGraph()
	proc := nn.ReifyForTraining(model, g).(*Model)

	// == Forward
	sequence := []ag.Node{
		g.NewVariable(mat.NewVecDense([]mat.Float{0.5234, 0.8113}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-1.7743, -0.5153}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{0.9396, -1.6837}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.2936, 0.3536}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.5461, 0.6564}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.7707, -1.346}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{1.8908, 0.2333}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.1716, -0.8175}), true),
	}

	y := proc.Forward(sequence...)
	assert.InDeltaSlice(t, []mat.Float{0.39854191, -0.017654827}, y[0].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.55043263, 0.1905072731}, y[1].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.22579388, -0.2080387961}, y[2].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.28493634, -0.1827686560}, y[3].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.61065015, 0.2028816025}, y[4].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.57294759, 0.174746066}, y[5].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.21875569, -0.210059952}, y[6].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.33201670, -0.082668955}, y[7].Value().Data(), 1.0e-05)
}

func TestModel_ForwardDownsampled(t *testing.T) {
	model := newTestModel()
	model.Config.DownsampleFactor = 4
	g := ag.NewGraph()
	proc := nn.ReifyForTraining(model, g).(*Model)

	// == Forward
	sequence := []ag.Node{
		g.NewVariable(mat.NewVecDense([]mat.Float{0.5234, 0.8113}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-1.7743, -0.5153}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{0.9396, -1.6837}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.2936, 0.3536}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.5461, 0.6564}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.7707, -1.346}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{1.8908, 0.2333}), true),
		g.NewVariable(mat.NewVecDense([]mat.Float{-0.1716, -0.8175}), true),
	}

	y := proc.Forward(sequence...)
	assert.InDeltaSlice(t, []mat.Float{0.3733543, -0.043746825}, y[0].Value().Data(), 1.0e-05)
	assert.InDeltaSlice(t, []mat.Float{0.4379011, 0.0260435049}, y[1].Value().Data(), 1.0e-05)
}

func newTestModel() *Model {
	c := Config{
		InputSize:               2,
		MaxBlockSize:            4,
		BlockSize:               []int{1, 2, 3, 4},
		DownsampleFactor:        1,
		ScoreConsensusAttention: false,
	}
	model := New(c)
	model.Conv[0].K[0].Value().SetData([]mat.Float{-0.4808, 0.1073, 0.4607, 0.0709})
	model.Conv[1].K[0].Value().SetData([]mat.Float{0.3772, 0.3505, -0.0531, -0.2144})
	model.Conv[0].B[0].Value().SetData([]mat.Float{0.4983})
	model.Conv[1].B[0].Value().SetData([]mat.Float{0.0743})
	model.Proj[0].K[0].Value().SetData([]mat.Float{0.4217, -0.2454})
	model.Proj[1].K[0].Value().SetData([]mat.Float{0.458, -0.426})
	model.Proj[0].B[0].Value().SetData([]mat.Float{0.1281})
	model.Proj[1].B[0].Value().SetData([]mat.Float{-0.3309})
	model.Scorer.W.Value().SetData([]mat.Float{-0.1683, 0.3895})
	model.Scorer.B.Value().SetData([]mat.Float{-0.2898})
	return model
}


<|start_filename|>pkg/ml/nn/convolution1d/model_test.go<|end_filename|>
// Copyright 2019 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package convolution1d

import (
	mat "github.com/nlpodyssey/spago/pkg/mat32"
	"github.com/nlpodyssey/spago/pkg/ml/ag"
	"github.com/nlpodyssey/spago/pkg/ml/nn"
	"github.com/stretchr/testify/assert"
	"testing"
)

//gocyclo:ignore
func TestModel_Forward(t *testing.T) {
	model := newTestModel()
	g := ag.NewGraph()

	// == Forward

	x1 := g.NewVariable(mat.NewDense(2, 4, []mat.Float{
		0.2, 0.1, 0.5, 0.8,
		0.4, -0.3, -0.2, -0.3,
	}), true)

	x2 := g.NewVariable(mat.NewDense(2, 4, []mat.Float{
		-0.2, 0.1, 0.5, 0.8,
		0.4, -0.3, -0.2, -0.9,
	}), true)

	x3 := g.NewVariable(mat.NewDense(2, 4, []mat.Float{
		0.2, 0.5, 0.9, 0.8,
		0.4, -0.5, -0.3, -0.2,
	}), true)

	y := nn.ReifyForTraining(model, g).(*Model).Forward(x1, x2, x3)

	assert.InDeltaSlice(t, []mat.Float{
		0.62914516, 0.42189900, 0.03997868,
	}, y[0].Value().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.77788806, 0.9775871, 0.99681227,
	}, y[1].Value().Data(), 1.0e-05)

	y[0].PropagateGrad(mat.NewDense(1, 3, []mat.Float{
		-0.3, 0.5, 0.6,
	}))

	y[1].PropagateGrad(mat.NewDense(1, 3, []mat.Float{
		-0.3, 0.5, -0.6,
	}))

	g.BackwardAll()

	assert.InDeltaSlice(t, []mat.Float{
		0.30437, 0.66660786,
		-0.31560957, -0.20753658,
	}, model.K[0].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.37687117, 0.66660786,
		-0.31560957, -0.56696117,
	}, model.K[1].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.70838666, 0.7585069,
		-0.4577138, -0.15248194,
	}, model.K[2].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.82878876,
	}, model.B[0].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.82878876,
	}, model.B[1].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.82878876,
	}, model.B[2].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		-0.023386842, -0.0038212487,
		-0.05327147, 0.032253552,
	}, model.K[3].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.023999983, -0.0038212487,
		-0.05327147, 0.03454506,
	}, model.K[4].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		-0.016049873, -0.04234343,
		-0.057321873, 0.053348884,
	}, model.K[5].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		-0.10012464,
	}, model.B[3].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		-0.10012464,
	}, model.B[4].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		-0.10012464,
	}, model.B[5].Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		-0.13801329, 0.19209248, 0.15132189, -0.24267177,
		0.052244477, 0.0015920401, 0.3153144, 0.17818466,
	}, x1.Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.09062646, -0.3191097, -0.16513953, 0.17780274,
		-0.07179071, -0.015045494, 0.4774822, 0.54104656,
	}, x2.Grad().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		-0.108041294, 0.045592614, 0.37506783, 0.17742082,
		-0.059944, -0.11689296, 0.38337404, 0.35636932,
	}, x3.Grad().Data(), 1.0e-05)
}

func TestDepthwise_Forward(t *testing.T) {
	model := newTestModel2()
	g := ag.NewGraph()

	// == Forward

	x1 := g.NewVariable(mat.NewDense(2, 4, []mat.Float{
		0.2, 0.1, 0.5, 0.8,
		0.4, -0.3, -0.2, -0.3,
	}), true)

	x2 := g.NewVariable(mat.NewDense(2, 4, []mat.Float{
		-0.2, 0.1, 0.5, 0.8,
		0.4, -0.3, -0.2, -0.9,
	}), true)

	x3 := g.NewVariable(mat.NewDense(2, 4, []mat.Float{
		0.2, 0.5, 0.9, 0.8,
		0.4, -0.5, -0.3, -0.2,
	}), true)

	y := nn.ReifyForTraining(model, g).(*Model).Forward(x1, x2, x3)

	assert.InDeltaSlice(t, []mat.Float{
		0.09, -0.3, -0.22,
	}, y[0].Value().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.14, 0.06, -0.66,
	}, y[1].Value().Data(), 1.0e-05)

	assert.InDeltaSlice(t, []mat.Float{
		0.51, 0.69, 0.92,
	}, y[2].Value().Data(), 1.0e-05)
}

func newTestModel() *Model {
	model := New(Config{
		KernelSizeX:    2,
		KernelSizeY:    2,
		YStride:        1,
		InputChannels:  3,
		OutputChannels: 2,
		Mask:           []int{1, 1, 1},
		DepthWise:      false,
		Activation:     ag.OpTanh,
	})
	model.K[0].Value().SetData([]mat.Float{
		0.5, -0.4,
		0.3, 0.3,
	})
	model.K[1].Value().SetData([]mat.Float{
		-0.5, 0.3,
		0.2, 0.9,
	})
	model.K[2].Value().SetData([]mat.Float{
		0.4, 0.3,
		0.2, 0.6,
	})
	model.B[0].Value().SetData([]mat.Float{0.0})
	model.B[1].Value().SetData([]mat.Float{0.2})
	model.B[2].Value().SetData([]mat.Float{0.5})
	model.K[3].Value().SetData([]mat.Float{
		0.4, 0.8,
		-0.9, 0.4,
	})
	model.K[4].Value().SetData([]mat.Float{
		0.0, 0.5,
		0.3, -0.5,
	})
	model.K[5].Value().SetData([]mat.Float{
		0.3, 0.6,
		0.2, 0.8,
	})
	model.B[3].Value().SetData([]mat.Float{0.4})
	model.B[4].Value().SetData([]mat.Float{0.1})
	model.B[5].Value().SetData([]mat.Float{0.5})
	return model
}

func newTestModel2() *Model {
	model := New(Config{
		KernelSizeX:    2,
		KernelSizeY:    2,
		YStride:        1,
		InputChannels:  3,
		OutputChannels: 3,
		Mask:           []int{1, 1, 1},
		DepthWise:      true,
		Activation:     ag.OpIdentity,
	})
	model.K[0].Value().SetData([]mat.Float{
		0.5, -0.4,
		0.3, 0.3,
	})
	model.K[1].Value().SetData([]mat.Float{
		-0.5, 0.3,
		0.2, 0.9,
	})
	model.K[2].Value().SetData([]mat.Float{
		0.4, 0.3,
		0.2, 0.6,
	})
	model.B[0].Value().SetData([]mat.Float{0.0})
	model.B[1].Value().SetData([]mat.Float{0.2})
	model.B[2].Value().SetData([]mat.Float{0.5})
	return model
}


<|start_filename|>pkg/mat64/mat64_test.go<|end_filename|>
// Copyright 2021 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"github.com/stretchr/testify/assert"
	"math"
	"testing"
)

func TestPow(t *testing.T) {
	assert.InDelta(t, Float(8), Pow(2, 3), 1e-10)
}

func TestCos(t *testing.T) {
	assert.InDelta(t, Float(-1), Cos(Pi), 1e-10)
}

func TestSin(t *testing.T) {
	assert.InDelta(t, Float(1), Sin(Pi/2), 1e-10)
}

func TestCosh(t *testing.T) {
	assert.InDelta(t, 11.59195, Cosh(Pi), 1e-5)
}

func TestSinh(t *testing.T) {
	assert.InDelta(t, 11.54874, Sinh(Pi), 1e-5)
}

func TestExp(t *testing.T) {
	assert.InDelta(t, 2.71828, Exp(1), 1e-5)
}

func TestAbs(t *testing.T) {
	assert.Equal(t, Float(42), Abs(42))
	assert.Equal(t, Float(42), Abs(-42))
}

func TestSqrt(t *testing.T) {
	assert.InDelta(t, Float(3), Sqrt(9), 1e-10)
}

func TestLog(t *testing.T) {
	assert.InDelta(t, 0.69314, Log(2), 1e-5)
}

func TestTan(t *testing.T) {
	assert.InDelta(t, Float(1), Tan(Pi/4), 1e-10)
}

func TestTanh(t *testing.T) {
	assert.InDelta(t, 0.76159, Tanh(1), 1e-5)
}

func TestMax(t *testing.T) {
	assert.Equal(t, Float(2), Max(1, 2))
	assert.Equal(t, Float(2), Max(2, 1))
}

func TestInf(t *testing.T) {
	assert.True(t, math.IsInf(Inf(1), +1))
	assert.True(t, math.IsInf(Inf(-1), -1))
}

func TestIsInf(t *testing.T) {
	assert.True(t, IsInf(Inf(1), +1))
	assert.True(t, IsInf(Inf(-1), -1))
	assert.False(t, IsInf(Inf(-1), +1))
	assert.False(t, IsInf(Inf(1), -1))
	assert.False(t, IsInf(Float(0), 1))
	assert.False(t, IsInf(Float(0), -1))
}

func TestNaN(t *testing.T) {
	assert.True(t, math.IsNaN(NaN()))
}

func TestCeil(t *testing.T) {
	assert.Equal(t, Float(2), Ceil(1.2))
}

func TestFloor(t *testing.T) {
	assert.Equal(t, Float(1), Floor(1.9))
}

func TestRound(t *testing.T) {
	assert.Equal(t, Float(1), Round(1.4))
	assert.Equal(t, Float(2), Round(1.5))
	assert.Equal(t, Float(2), Round(1.6))
}


<|start_filename|>pkg/mat32/mat32.go<|end_filename|>
// Copyright 2021 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat32

import (
	"github.com/nlpodyssey/spago/pkg/mat32/internal/math32"
	"math"
)

// Float is the main float type for the mat32 package. It is an alias for float32.
type Float = float32

const (
	// SmallestNonzeroFloat corresponds to math.SmallestNonzeroFloat32.
	SmallestNonzeroFloat = Float(math.SmallestNonzeroFloat32)
	// Pi mathematical constant.
	Pi = Float(math.Pi)
)

// Pow returns x**y, the base-x exponential of y.
func Pow(x, y Float) Float {
	return math32.Pow(x, y)
}

// Cos returns the cosine of the radian argument x.
func Cos(x Float) Float {
	return Float(math.Cos(float64(x)))
}

// Sin returns the sine of the radian argument x.
func Sin(x Float) Float {
	return Float(math.Sin(float64(x)))
}

// Cosh returns the hyperbolic cosine of x.
func Cosh(x Float) Float {
	return Float(math.Cosh(float64(x)))
}

// Sinh returns the hyperbolic sine of x.
func Sinh(x Float) Float {
	return Float(math.Sinh(float64(x)))
}

// Exp returns e**x, the base-e exponential of x.
func Exp(x Float) Float {
	return math32.Exp(x)
}

// Abs returns the absolute value of x.
func Abs(x Float) Float {
	return math32.Abs(x)
}

// Sqrt returns the square root of x.
func Sqrt(x Float) Float {
	return math32.Sqrt(x)
}

// Log returns the natural logarithm of x.
func Log(x Float) Float {
	return math32.Log(x)
}

// Tan returns the tangent of the radian argument x.
func Tan(x Float) Float {
	return Float(math.Tan(float64(x)))
}

// Tanh returns the hyperbolic tangent of x.
func Tanh(x Float) Float {
	return math32.Tanh(x)
}

// Max returns the larger of x or y.
func Max(x, y Float) Float {
	return math32.Max(x, y)
}

// Inf returns positive infinity if sign >= 0, negative infinity if sign < 0.
func Inf(sign int) Float {
	return math32.Inf(sign)
}

// IsInf reports whether f is an infinity, according to sign.
func IsInf(f Float, sign int) bool {
	return math32.IsInf(f, sign)
}

// NaN returns an IEEE 754 ``not-a-number'' value.
func NaN() Float {
	return math32.NaN()
}

// Ceil returns the least integer value greater than or equal to x.
func Ceil(x Float) Float {
	return math32.Ceil(x)
}

// Floor returns the greatest integer value less than or equal to x.
func Floor(x Float) Float {
	return math32.Floor(x)
}

// Round returns the nearest integer, rounding half away from zero.
func Round(x Float) Float {
	return Float(math.Round(float64(x)))
}


<|start_filename|>pkg/ml/nn/param_encoding.go<|end_filename|>
// Copyright 2021 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package nn

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"fmt"
	mat "github.com/nlpodyssey/spago/pkg/mat32"
	"io"
	"log"
)

// init registers the param implementation with the gob subsystem - so that it knows how to encode and decode
// values of type nn.Param
func init() {
	gob.Register(&param{})
}

// MarshalBinary marshals a param into binary form.
func (r *param) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)

	err := mat.MarshalBinaryMatrix(r.value, buf)
	if err != nil {
		return nil, err
	}

	if r.payload == nil {
		buf.WriteByte(0)
	} else {
		buf.WriteByte(1)
		pBin, err := r.payload.MarshalBinary()
		if err != nil {
			return nil, err
		}
		buf.Write(pBin)
	}

	return buf.Bytes(), nil
}

// UnmarshalBinary unmarshals a param from binary form.
func (r *param) UnmarshalBinary(data []byte) error {
	var err error
	buf := bytes.NewReader(data)

	r.value, err = mat.UnmarshalBinaryMatrix(buf)
	if err != nil {
		return err
	}

	hasPayload, err := buf.ReadByte()
	if hasPayload == 0 {
		r.payload = nil
		return nil
	}

	r.payload = new(Payload)

	pBin, err := io.ReadAll(buf)
	if err != nil {
		return err
	}

	return r.payload.UnmarshalBinary(pBin)
}

// MarshalBinaryParam encodes a Param into binary form.
func MarshalBinaryParam(p Param, w io.Writer) error {
	if p == nil {
		_, err := w.Write([]byte{0})
		return err
	}

	pp, isParam := p.(*param)
	if !isParam {
		log.Fatal(fmt.Errorf("unsupported Param implementation for binary marshaling, %T: %#v", p, p))
	}

	_, err := w.Write([]byte{1})
	if err != nil {
		return err
	}

	bin, err := pp.MarshalBinary()
	if err != nil {
		return err
	}

	binLen := make([]byte, 4)
	binary.LittleEndian.PutUint32(binLen, uint32(len(bin)))
	_, err = w.Write(binLen)
	if err != nil {
		return err
	}

	_, err = w.Write(bin)
	return err
}

// UnmarshalBinaryParam decodes a Param from binary form.
// TODO: add a "withBacking" optional argument to remove the need of UnmarshalBinaryParamWithReceiver().
func UnmarshalBinaryParam(r io.Reader) (Param, error) {
	isPresent := make([]byte, 1)
	_, err := r.Read(isPresent)
	if err != nil {
		return nil, err
	}
	if isPresent[0] == 0 {
		return nil, nil
	}

	binLenBytes := make([]byte, 4)
	_, err = r.Read(binLenBytes)
	if err != nil {
		return nil, err
	}
	binLen := int(binary.LittleEndian.Uint32(binLenBytes))
	bin := make([]byte, binLen)
	_, err = r.Read(bin)
	if err != nil {
		return nil, err
	}

	p := new(param)
	err = p.UnmarshalBinary(bin)
	if err != nil {
		return nil, err
	}
	return p, nil
}

// UnmarshalBinaryParamWithReceiver decodes a Param from binary form into the receiver.
func UnmarshalBinaryParamWithReceiver(r io.Reader, dest Param) error {
	p, isParam := dest.(*param)
	if !isParam {
		log.Fatal(fmt.Errorf("unsupported Param implementation for binary unmarshaling, %T: %#v", p, p))
	}

	isPresent := make([]byte, 1)
	_, err := r.Read(isPresent)
	if err != nil {
		return err
	}
	if isPresent[0] == 0 {
		return nil
	}

	binLenBytes := make([]byte, 4)
	_, err = r.Read(binLenBytes)
	if err != nil {
		return err
	}
	binLen := int(binary.LittleEndian.Uint32(binLenBytes))
	bin := make([]byte, binLen)
	_, err = r.Read(bin)
	if err != nil {
		return err
	}

	err = p.UnmarshalBinary(bin)
	if err != nil {
		return err
	}
	return nil
}


<|start_filename|>pkg/mat64/internal/gonum.go<|end_filename|>
// Copyright ©2016-2017 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package internal

import "github.com/nlpodyssey/spago/pkg/mat64/internal/asm/f64"

// MatrixMul computes the matrix-matrix multiplication C = A * B.
// This code is adapted from Gonum's Dgemm implementation.
func MatrixMul(aRows, aCols, bCols int, a []float64, b []float64, c []float64) {
	for i := 0; i < aRows; i++ {
		ctmp := c[i*bCols : i*bCols+bCols]
		for l, v := range a[i*aCols : i*aCols+aCols] {
			if v == 0 {
				continue
			}
			f64.AxpyUnitary(v, b[l*bCols:l*bCols+bCols], ctmp)
		}
	}
}


<|start_filename|>pkg/ml/optimizers/gd/gdmbuilder/gdbuilder.go<|end_filename|>
// Copyright 2019 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gdmbuilder

import (
	"github.com/nlpodyssey/spago/pkg/ml/optimizers/gd"
	"github.com/nlpodyssey/spago/pkg/ml/optimizers/gd/adagrad"
	"github.com/nlpodyssey/spago/pkg/ml/optimizers/gd/adam"
	"github.com/nlpodyssey/spago/pkg/ml/optimizers/gd/lamb"
	"github.com/nlpodyssey/spago/pkg/ml/optimizers/gd/radam"
	"github.com/nlpodyssey/spago/pkg/ml/optimizers/gd/rmsprop"
	"github.com/nlpodyssey/spago/pkg/ml/optimizers/gd/sgd"
)

// NewMethod returns a new gd.Method, chosen and initialized according to
// the given config.
// It panics if the config type is unknown or unsupported.
func NewMethod(config gd.MethodConfig) gd.Method {
	switch config := config.(type) {
	case adagrad.Config:
		return adagrad.New(config)
	case adam.Config:
		return adam.New(config)
	case radam.Config:
		return radam.New(config)
	case rmsprop.Config:
		return rmsprop.New(config)
	case lamb.Config:
		return lamb.New(config)
	case sgd.Config:
		return sgd.New(config)
	default:
		panic("gd: unknown method configuration")
	}
}


<|start_filename|>pkg/mat32/internal/math32/exp.go<|end_filename|>
package math32

// Exp returns e**x, the base-e exponential of x.
func Exp(float32) float32

func exp(x float32) float32 {
	const (
		Ln2Hi = float32(6.9313812256e-01)
		Ln2Lo = float32(9.0580006145e-06)
		Log2e = float32(1.4426950216e+00)

		Overflow  = 7.09782712893383973096e+02
		Underflow = -7.45133219101941108420e+02
		NearZero  = 1.0 / (1 << 28) // 2**-28
	)
	// hx := Float32bits(x) & uint32(0x7fffffff)

	// special cases
	switch {
	case IsNaN(x) || IsInf(x, 1):
		return x
	case IsInf(x, -1):
		return 0
	case x > Overflow:
		return Inf(1)
	case x < Underflow:
		return 0
		// case hx > LogMax:
		// 	return Inf(1)
		// case x < 0 && hx > LogMin:
		//  return 0
	case -NearZero < x && x < NearZero:
		return 1 + x
	}

	// reduce; computed as r = hi - lo for extra precision.
	var k int
	switch {
	case x < 0:
		k = int(Log2e*x - 0.5)
	case x > 0:
		k = int(Log2e*x + 0.5)
	}
	hi := x - float32(k)*Ln2Hi
	lo := float32(k) * Ln2Lo

	// compute
	return expmulti(hi, lo, k)
}

// exp1 returns e**r × 2**k where r = hi - lo and |r| ≤ ln(2)/2.
func expmulti(hi, lo float32, k int) float32 {
	const (
		P1 = float32(1.6666667163e-01)  /* 0x3e2aaaab */
		P2 = float32(-2.7777778450e-03) /* 0xbb360b61 */
		P3 = float32(6.6137559770e-05)  /* 0x388ab355 */
		P4 = float32(-1.6533901999e-06) /* 0xb5ddea0e */
		P5 = float32(4.1381369442e-08)  /* 0x3331bb4c */
	)

	r := hi - lo
	t := r * r
	c := r - t*(P1+t*(P2+t*(P3+t*(P4+t*P5))))
	y := 1 - ((lo - (r*c)/(2-c)) - hi)
	// TODO(rsc): make sure Ldexp can handle boundary k
	return Ldexp(y, k)
}


<|start_filename|>pkg/utils/kvdb/kvdb_test.go<|end_filename|>
// Copyright 2019 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package kvdb

import (
	"bytes"
	"encoding/gob"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"os"
	"testing"
)

func TestKeyValueDB_ReadOnlyAndForceNew(t *testing.T) {
	t.Parallel()

	dir := newTempDir(t, "spago-kvdb-test-")
	defer os.RemoveAll(dir)

	// Create a new DB and write something
	db1 := NewDefaultKeyValueDB(Config{Path: dir, ReadOnly: false, ForceNew: true})

	err := db1.Put([]byte{1}, []byte{2})
	require.Nil(t, err)

	err = db1.Close()
	require.Nil(t, err)

	// Reopen the same DB in read-only mode
	db2 := NewDefaultKeyValueDB(Config{Path: dir, ReadOnly: true, ForceNew: false})

	value, ok, err := db2.Get([]byte{1})
	require.Nil(t, err)
	assert.True(t, ok)
	assert.Equal(t, []byte{2}, value)

	err = db2.Put([]byte{3}, []byte{4})
	assert.NotNil(t, err)
	value, ok, err = db2.Get([]byte{3})
	require.Nil(t, err)
	assert.False(t, ok)
	assert.Nil(t, value)

	err = db2.Close()
	require.Nil(t, err)

	// Reopen a new DB reusing the same folder and forcing cleanup
	db3 := NewDefaultKeyValueDB(Config{Path: dir, ReadOnly: false, ForceNew: true})

	value, ok, err = db3.Get([]byte{1})
	require.Nil(t, err)
	assert.False(t, ok)
	assert.Nil(t, value)

	err = db3.Close()
	require.Nil(t, err)
}

func TestKeyValueDB_DropAll(t *testing.T) {
	t.Parallel()

	dir := newTempDir(t, "spago-kvdb-test-")
	defer os.RemoveAll(dir)

	db := NewDefaultKeyValueDB(Config{Path: dir, ReadOnly: false, ForceNew: true})
	defer db.Close()

	err := db.Put([]byte{1}, []byte{2})
	require.Nil(t, err)
	err = db.Put([]byte{3}, []byte{4})
	require.Nil(t, err)

	keys, err := db.Keys()
	require.Nil(t, err)
	assert.NotEmpty(t, keys)

	err = db.DropAll()
	require.Nil(t, err)

	keys, err = db.Keys()
	require.Nil(t, err)
	assert.Empty(t, keys)
}

func TestKeyValueDB_Keys(t *testing.T) {
	t.Parallel()

	dir := newTempDir(t, "spago-kvdb-test-")
	defer os.RemoveAll(dir)

	db := NewDefaultKeyValueDB(Config{Path: dir, ReadOnly: false, ForceNew: true})
	defer db.Close()

	err := db.Put([]byte{1}, []byte{2})
	require.Nil(t, err)

}

func TestKeyValueDB_PutAndGet(t *testing.T) {
	t.Parallel()

	dir := newTempDir(t, "spago-kvdb-test-")
	defer os.RemoveAll(dir)

	db := NewDefaultKeyValueDB(Config{Path: dir, ReadOnly: false, ForceNew: true})
	defer db.Close()

	err := db.Put([]byte{1}, []byte{2})
	require.Nil(t, err)

	// Existing key
	value, ok, err := db.Get([]byte{1})
	require.Nil(t, err)
	assert.True(t, ok)
	assert.Equal(t, []byte{2}, value)

	// Nonexistent key
	value, ok, err = db.Get([]byte{9})
	require.Nil(t, err)
	assert.False(t, ok)
	assert.Nil(t, value)
}

func TestKeyValueDB_Gob(t *testing.T) {
	t.Parallel()

	var buf bytes.Buffer

	// Setup first DB
	dir1 := newTempDir(t, "spago-kvdb-test-1-")
	defer os.RemoveAll(dir1)

	db1 := NewDefaultKeyValueDB(Config{Path: dir1, ReadOnly: false, ForceNew: true})
	defer db1.Close()
	err := db1.Put([]byte{42}, []byte{1})
	require.Nil(t, err)

	// Setup second DB
	dir2 := newTempDir(t, "spago-kvdb-test-2-")
	defer os.RemoveAll(dir2)

	db2 := NewDefaultKeyValueDB(Config{Path: dir2, ReadOnly: false, ForceNew: true})
	defer db2.Close()
	err = db2.Put([]byte{42}, []byte{2})
	require.Nil(t, err)

	// Encode first DB
	enc := gob.NewEncoder(&buf)
	err = enc.Encode(&db1)
	require.Nil(t, err)

	// Decode to second DB
	dec := gob.NewDecoder(&buf)
	err = dec.Decode(&db2)
	require.Nil(t, err)

	// First DB: still points to dir1 and contains 42 => 1
	assert.Equal(t, dir1, db1.Config.Path)
	v1, _, err := db1.Get([]byte{42})
	require.Nil(t, err)
	assert.Equal(t, []byte{1}, v1)

	// Second DB: it should be unmodified, pointing to dir2 and containing 42 => 2
	assert.Equal(t, dir2, db2.Config.Path)
	v2, _, err := db2.Get([]byte{42})
	require.Nil(t, err)
	assert.Equal(t, []byte{2}, v2)
}

func newTempDir(t *testing.T, pattern string) string {
	t.Helper()
	dir, err := os.MkdirTemp("", pattern)
	require.Nil(t, err)
	return dir
}


<|start_filename|>pkg/ml/optimizers/gd/lamb/lamb.go<|end_filename|>
// Copyright 2021 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package lamb

import (
	mat "github.com/nlpodyssey/spago/pkg/mat32"
	"github.com/nlpodyssey/spago/pkg/ml/nn"
	"github.com/nlpodyssey/spago/pkg/ml/optimizers/gd"
)

var _ gd.MethodConfig = &Config{}

// Config provides configuration settings for Lamb optimizer.
type Config struct {
	gd.MethodConfig
	StepSize mat.Float
	Beta1    mat.Float
	Beta2    mat.Float
	Epsilon  mat.Float
	Lambda   mat.Float
}

// NewConfig returns a new Lamb Config.
func NewConfig(stepSize, beta1, beta2, epsilon, lambda mat.Float) Config {
	if !(beta1 >= 0.0 && beta1 < 1.0) {
		panic("lamb: `beta1` must be in the range [0.0, 1.0)")
	}
	if !(beta2 >= 0.0 && beta2 < 1.0) {
		panic("lamb: `beta2` must be in the range [0.0, 1.0)")
	}
	return Config{
		StepSize: stepSize,
		Beta1:    beta1,
		Beta2:    beta2,
		Epsilon:  epsilon,
		Lambda:   lambda,
	}
}

// NewDefaultConfig returns a new Config with generically reasonable default values.
func NewDefaultConfig() Config {
	return Config{
		StepSize: 0.001,
		Beta1:    0.9,
		Beta2:    0.999,
		Epsilon:  1.0e-8,
		Lambda:   0.1,
	}
}

var _ gd.Method = &Lamb{}

// Lamb implements the Lamb gradient descent optimization method.
type Lamb struct {
	Config
	Alpha    mat.Float
	TimeStep int
}

// New returns a new Lamb optimizer, initialized according to the given configuration.
func New(c Config) *Lamb {
	lamb := &Lamb{
		Config: c,
		Alpha:  c.StepSize,
	}
	lamb.IncExample() // initialize 'alpha' coefficient
	return lamb
}

// Label returns the enumeration-like value which identifies this gradient descent method.
func (o *Lamb) Label() int {
	return gd.Lamb
}

const (
	v    int = 0
	m    int = 1
	buf1 int = 2 // contains 'grads.ProdScalar(1.0 - beta1)'
	buf2 int = 3 // contains 'grads.Prod(grads).ProdScalar(1.0 - beta2)'
	buf3 int = 4
)

// NewSupport returns a new support structure with the given dimensions.
func (o *Lamb) NewSupport(r, c int) *nn.Payload {
	supp := make([]mat.Matrix, 5)
	supp[v] = mat.NewEmptyDense(r, c)
	supp[m] = mat.NewEmptyDense(r, c)
	supp[buf1] = mat.NewEmptyDense(r, c)
	supp[buf2] = mat.NewEmptyDense(r, c)
	supp[buf3] = mat.NewEmptyDense(r, c)
	return &nn.Payload{
		Label: o.Label(),
		Data:  supp,
	}
}

// IncExample beats the occurrence of a new example.
func (o *Lamb) IncExample() {
	o.TimeStep++
	o.updateAlpha()
}

func (o *Lamb) updateAlpha() {
	o.Alpha = o.StepSize * mat.Sqrt(1.0-mat.Pow(o.Beta2, mat.Float(o.TimeStep))) / (1.0 - mat.Pow(o.Beta1, mat.Float(o.TimeStep)))
}

// Delta returns the difference between the current params and where the method wants it to be.
func (o *Lamb) Delta(param nn.Param) mat.Matrix {
	return o.calcDelta(param.Grad(), gd.GetOrSetPayload(param, o).Data, param.Value())
}

// v = v*beta1 + grads*(1.0-beta1)
// m = m*beta2 + (grads*grads)*(1.0-beta2)
// weights = ||params|| / || (v / (sqrt(m) + eps)) + (lambda * weights)
// d = (v / (sqrt(m) + eps)) + (lambda * weights) * alpha
func (o *Lamb) calcDelta(grads mat.Matrix, supp []mat.Matrix, weights mat.Matrix) mat.Matrix {
	updateV(grads, supp, o.Beta1)
	updateM(grads, supp, o.Beta2)
	buf := supp[m].Sqrt().AddScalarInPlace(o.Epsilon)
	defer mat.ReleaseMatrix(buf)
	suppDiv := supp[v].Div(buf)
	if o.Lambda != 0.0 {
		scaledW := weights.ProdScalar(o.Lambda)
		suppDiv.AddInPlace(scaledW)
	}
	weightsNorm := norm(weights)
	adamStepNorm := norm(suppDiv)
	trustRatio := mat.Float(1.0)
	if !(weightsNorm == 0.0 || adamStepNorm == 0.0) {
		trustRatio = weightsNorm / adamStepNorm
	}
	defer mat.ReleaseMatrix(suppDiv)
	supp[buf3].ProdMatrixScalarInPlace(suppDiv, o.Alpha*trustRatio)
	return supp[buf3]
}

// v = v*beta1 + grads*(1.0-beta1)
func updateV(grads mat.Matrix, supp []mat.Matrix, beta1 mat.Float) {
	supp[v].ProdScalarInPlace(beta1)
	supp[buf1].ProdMatrixScalarInPlace(grads, 1.0-beta1)
	supp[v].AddInPlace(supp[buf1])
}

// m = m*beta2 + (grads*grads)*(1.0-beta2)
func updateM(grads mat.Matrix, supp []mat.Matrix, beta2 mat.Float) {
	supp[m].ProdScalarInPlace(beta2)
	sqGrad := grads.Prod(grads)
	defer mat.ReleaseMatrix(sqGrad)
	supp[buf2].ProdMatrixScalarInPlace(sqGrad, 1.0-beta2)
	supp[m].AddInPlace(supp[buf2])
}

func norm(grads mat.Matrix) mat.Float {
	sum := mat.Float(0.0)
	for _, d := range grads.Data() {
		sum += d * d
	}
	return mat.Sqrt(sum)
}


<|start_filename|>pkg/mat32/internal/math32/all_test.go<|end_filename|>
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package math32_test

import (
	"fmt"
	"testing"

	. "github.com/nlpodyssey/spago/pkg/mat32/internal/math32"
)

var vf = []float32{
	4.9790119248836735e+00,
	7.7388724745781045e+00,
	-2.7688005719200159e-01,
	-5.0106036182710749e+00,
	9.6362937071984173e+00,
	2.9263772392439646e+00,
	5.2290834314593066e+00,
	2.7279399104360102e+00,
	1.8253080916808550e+00,
	-8.6859247685756013e+00,
}

// The expected results below were computed by the high precision calculators
// at http://keisan.casio.com/.  More exact input values (array vf[], above)
// were obtained by printing them with "%.26f".  The answers were calculated
// to 26 digits (by using the "Digit number" drop-down control of each
// calculator).

var ceil = []float32{
	5.0000000000000000e+00,
	8.0000000000000000e+00,
	0.0000000000000000e+00,
	-5.0000000000000000e+00,
	1.0000000000000000e+01,
	3.0000000000000000e+00,
	6.0000000000000000e+00,
	3.0000000000000000e+00,
	2.0000000000000000e+00,
	-8.0000000000000000e+00,
}
var copysign = []float32{
	-4.9790119248836735e+00,
	-7.7388724745781045e+00,
	-2.7688005719200159e-01,
	-5.0106036182710749e+00,
	-9.6362937071984173e+00,
	-2.9263772392439646e+00,
	-5.2290834314593066e+00,
	-2.7279399104360102e+00,
	-1.8253080916808550e+00,
	-8.6859247685756013e+00,
}

var exp = []float32{
	1.4533071302642137507696589e+02,
	2.2958822575694449002537581e+03,
	7.5814542574851666582042306e-01,
	6.6668778421791005061482264e-03,
	1.5310493273896033740861206e+04,
	1.8659907517999328638667732e+01,
	1.8662167355098714543942057e+02,
	1.5301332413189378961665788e+01,
	6.2047063430646876349125085e+00,
	1.6894712385826521111610438e-04,
}
var fabs = []float32{
	4.9790119248836735e+00,
	7.7388724745781045e+00,
	2.7688005719200159e-01,
	5.0106036182710749e+00,
	9.6362937071984173e+00,
	2.9263772392439646e+00,
	5.2290834314593066e+00,
	2.7279399104360102e+00,
	1.8253080916808550e+00,
	8.6859247685756013e+00,
}
var fdim = []float32{
	4.9790119248836735e+00,
	7.7388724745781045e+00,
	0.0000000000000000e+00,
	0.0000000000000000e+00,
	9.6362937071984173e+00,
	2.9263772392439646e+00,
	5.2290834314593066e+00,
	2.7279399104360102e+00,
	1.8253080916808550e+00,
	0.0000000000000000e+00,
}
var floor = []float32{
	4.0000000000000000e+00,
	7.0000000000000000e+00,
	-1.0000000000000000e+00,
	-6.0000000000000000e+00,
	9.0000000000000000e+00,
	2.0000000000000000e+00,
	5.0000000000000000e+00,
	2.0000000000000000e+00,
	1.0000000000000000e+00,
	-9.0000000000000000e+00,
}

type fi struct {
	f float32
	i int
}

var frexp = []fi{
	{6.2237649061045918750e-01, 3},
	{9.6735905932226306250e-01, 3},
	{-5.5376011438400318000e-01, -1},
	{-6.2632545228388436250e-01, 3},
	{6.02268356699901081250e-01, 4},
	{7.3159430981099115000e-01, 2},
	{6.5363542893241332500e-01, 3},
	{6.8198497760900255000e-01, 2},
	{9.1265404584042750000e-01, 1},
	{-5.4287029803597508250e-01, 4},
}
var log = []float32{
	1.605231462693062999102599e+00,
	2.0462560018708770653153909e+00,
	-1.2841708730962657801275038e+00,
	1.6115563905281545116286206e+00,
	2.2655365644872016636317461e+00,
	1.0737652208918379856272735e+00,
	1.6542360106073546632707956e+00,
	1.0035467127723465801264487e+00,
	6.0174879014578057187016475e-01,
	2.161703872847352815363655e+00,
}
var modf = [][2]float32{
	{4.0000000000000000e+00, 9.7901192488367350108546816e-01},
	{7.0000000000000000e+00, 7.3887247457810456552351752e-01},
	{Copysign(0, -1), -2.7688005719200159404635997e-01},
	{-5.0000000000000000e+00, -1.060361827107492160848778e-02},
	{9.0000000000000000e+00, 6.3629370719841737980004837e-01},
	{2.0000000000000000e+00, 9.2637723924396464525443662e-01},
	{5.0000000000000000e+00, 2.2908343145930665230025625e-01},
	{2.0000000000000000e+00, 7.2793991043601025126008608e-01},
	{1.0000000000000000e+00, 8.2530809168085506044576505e-01},
	{-8.0000000000000000e+00, -6.8592476857560136238589621e-01},
}

var pow = []float32{
	9.5282232631648411840742957e+04,
	5.4811599352999901232411871e+07,
	5.2859121715894396531132279e-01,
	9.7587991957286474464259698e-06,
	4.328064329346044846740467e+09,
	8.4406761805034547437659092e+02,
	1.6946633276191194947742146e+05,
	5.3449040147551939075312879e+02,
	6.688182138451414936380374e+01,
	2.0609869004248742886827439e-09,
}
var signbit = []bool{
	false,
	false,
	true,
	true,
	false,
	false,
	false,
	false,
	false,
	true,
}

var sqrt = []float32{
	2.2313699659365484748756904e+00,
	2.7818829009464263511285458e+00,
	5.2619393496314796848143251e-01,
	2.2384377628763938724244104e+00,
	3.1042380236055381099288487e+00,
	1.7106657298385224403917771e+00,
	2.286718922705479046148059e+00,
	1.6516476350711159636222979e+00,
	1.3510396336454586262419247e+00,
	2.9471892997524949215723329e+00,
}

var tanh = []float32{
	9.9990531206936338549262119e-01,
	9.9999962057085294197613294e-01,
	-2.7001505097318677233756845e-01,
	-9.9991110943061718603541401e-01,
	9.9999999146798465745022007e-01,
	9.9427249436125236705001048e-01,
	9.9994257600983138572705076e-01,
	9.9149409509772875982054701e-01,
	9.4936501296239685514466577e-01,
	-9.9999994291374030946055701e-01,
}
var trunc = []float32{
	4.0000000000000000e+00,
	7.0000000000000000e+00,
	-0.0000000000000000e+00,
	-5.0000000000000000e+00,
	9.0000000000000000e+00,
	2.0000000000000000e+00,
	5.0000000000000000e+00,
	2.0000000000000000e+00,
	1.0000000000000000e+00,
	-8.0000000000000000e+00,
}

// arguments and expected results for special cases

var vfceilSC = []float32{
	Inf(-1),
	Copysign(0, -1),
	0,
	Inf(1),
	NaN(),
}
var ceilSC = []float32{
	Inf(-1),
	Copysign(0, -1),
	0,
	Inf(1),
	NaN(),
}

var vfcopysignSC = []float32{
	Inf(-1),
	Inf(1),
	NaN(),
}
var copysignSC = []float32{
	Inf(-1),
	Inf(-1),
	NaN(),
}

var vfexpSC = []float32{
	Inf(-1),
	-2000,
	2000,
	Inf(1),
	NaN(),
}
var expSC = []float32{
	0,
	0,
	Inf(1),
	Inf(1),
	NaN(),
}

var vffabsSC = []float32{
	Inf(-1),
	Copysign(0, -1),
	0,
	Inf(1),
	NaN(),
}
var fabsSC = []float32{
	Inf(1),
	0,
	0,
	Inf(1),
	NaN(),
}

var vffdimSC = [][2]float32{
	{Inf(-1), Inf(-1)},
	{Inf(-1), Inf(1)},
	{Inf(-1), NaN()},
	{Copysign(0, -1), Copysign(0, -1)},
	{Copysign(0, -1), 0},
	{0, Copysign(0, -1)},
	{0, 0},
	{Inf(1), Inf(-1)},
	{Inf(1), Inf(1)},
	{Inf(1), NaN()},
	{NaN(), Inf(-1)},
	{NaN(), Copysign(0, -1)},
	{NaN(), 0},
	{NaN(), Inf(1)},
	{NaN(), NaN()},
}
var nan = Float32frombits(0x7FF80001) // SSE2 DIVSD 0/0
var vffdim2SC = [][2]float32{
	{Inf(-1), Inf(-1)},
	{Inf(-1), Inf(1)},
	{Inf(-1), nan},
	{Copysign(0, -1), Copysign(0, -1)},
	{Copysign(0, -1), 0},
	{0, Copysign(0, -1)},
	{0, 0},
	{Inf(1), Inf(-1)},
	{Inf(1), Inf(1)},
	{Inf(1), nan},
	{nan, Inf(-1)},
	{nan, Copysign(0, -1)},
	{nan, 0},
	{nan, Inf(1)},
	{nan, nan},
}
var fdimSC = []float32{
	NaN(),
	0,
	NaN(),
	0,
	0,
	0,
	0,
	Inf(1),
	NaN(),
	NaN(),
	NaN(),
	NaN(),
	NaN(),
	NaN(),
	NaN(),
}
var fmaxSC = []float32{
	Inf(-1),
	Inf(1),
	NaN(),
	Copysign(0, -1),
	0,
	0,
	0,
	Inf(1),
	Inf(1),
	Inf(1),
	NaN(),
	NaN(),
	NaN(),
	Inf(1),
	NaN(),
}
var fminSC = []float32{
	Inf(-1),
	Inf(-1),
	Inf(-1),
	Copysign(0, -1),
	Copysign(0, -1),
	Copysign(0, -1),
	0,
	Inf(-1),
	Inf(1),
	NaN(),
	Inf(-1),
	NaN(),
	NaN(),
	NaN(),
	NaN(),
}

var vffrexpSC = []float32{
	Inf(-1),
	Copysign(0, -1),
	0,
	Inf(1),
	NaN(),
}
var frexpSC = []fi{
	{Inf(-1), 0},
	{Copysign(0, -1), 0},
	{0, 0},
	{Inf(1), 0},
	{NaN(), 0},
}

var vfldexpSC = []fi{
	{0, 0},
	{0, -1075},
	{0, 1024},
	{Copysign(0, -1), 0},
	{Copysign(0, -1), -1075},
	{Copysign(0, -1), 1024},
	{Inf(1), 0},
	{Inf(1), -1024},
	{Inf(-1), 0},
	{Inf(-1), -1024},
	{NaN(), -1024},
}
var ldexpSC = []float32{
	0,
	0,
	0,
	Copysign(0, -1),
	Copysign(0, -1),
	Copysign(0, -1),
	Inf(1),
	Inf(1),
	Inf(-1),
	Inf(-1),
	NaN(),
}

var vflogSC = []float32{
	Inf(-1),
	-Pi,
	Copysign(0, -1),
	0,
	1,
	Inf(1),
	NaN(),
}
var logSC = []float32{
	NaN(),
	NaN(),
	Inf(-1),
	Inf(-1),
	0,
	Inf(1),
	NaN(),
}

var vfmodfSC = []float32{
	Inf(-1),
	Copysign(0, -1),
	Inf(1),
	NaN(),
}
var modfSC = [][2]float32{
	{Inf(-1), NaN()}, // [2]float32{Copysign(0, -1), Inf(-1)},
	{Copysign(0, -1), Copysign(0, -1)},
	{Inf(1), NaN()}, // [2]float32{0, Inf(1)},
	{NaN(), NaN()},
}

var vfpowSC = [][2]float32{
	{Inf(-1), -Pi},
	{Inf(-1), -3},
	{Inf(-1), Copysign(0, -1)},
	{Inf(-1), 0},
	{Inf(-1), 1},
	{Inf(-1), 3},
	{Inf(-1), Pi},
	{Inf(-1), NaN()},

	{-Pi, Inf(-1)},
	{-Pi, -Pi},
	{-Pi, Copysign(0, -1)},
	{-Pi, 0},
	{-Pi, 1},
	{-Pi, Pi},
	{-Pi, Inf(1)},
	{-Pi, NaN()},

	{-1, Inf(-1)},
	{-1, Inf(1)},
	{-1, NaN()},
	{-1 / 2, Inf(-1)},
	{-1 / 2, Inf(1)},
	{Copysign(0, -1), Inf(-1)},
	{Copysign(0, -1), -Pi},
	{Copysign(0, -1), -3},
	{Copysign(0, -1), 3},
	{Copysign(0, -1), Pi},
	{Copysign(0, -1), Inf(1)},

	{0, Inf(-1)},
	{0, -Pi},
	{0, -3},
	{0, Copysign(0, -1)},
	{0, 0},
	{0, 3},
	{0, Pi},
	{0, Inf(1)},
	{0, NaN()},

	{1 / 2, Inf(-1)},
	{1 / 2, Inf(1)},
	{1, Inf(-1)},
	{1, Inf(1)},
	{1, NaN()},

	{Pi, Inf(-1)},
	{Pi, Copysign(0, -1)},
	{Pi, 0},
	{Pi, 1},
	{Pi, Inf(1)},
	{Pi, NaN()},
	{Inf(1), -Pi},
	{Inf(1), Copysign(0, -1)},
	{Inf(1), 0},
	{Inf(1), 1},
	{Inf(1), Pi},
	{Inf(1), NaN()},
	{NaN(), -Pi},
	{NaN(), Copysign(0, -1)},
	{NaN(), 0},
	{NaN(), 1},
	{NaN(), Pi},
	{NaN(), NaN()},
}
var powSC = []float32{
	0,               // pow(-Inf, -Pi)
	Copysign(0, -1), // pow(-Inf, -3)
	1,               // pow(-Inf, -0)
	1,               // pow(-Inf, +0)
	Inf(-1),         // pow(-Inf, 1)
	Inf(-1),         // pow(-Inf, 3)
	Inf(1),          // pow(-Inf, Pi)
	NaN(),           // pow(-Inf, NaN)
	0,               // pow(-Pi, -Inf)
	NaN(),           // pow(-Pi, -Pi)
	1,               // pow(-Pi, -0)
	1,               // pow(-Pi, +0)
	-Pi,             // pow(-Pi, 1)
	NaN(),           // pow(-Pi, Pi)
	Inf(1),          // pow(-Pi, +Inf)
	NaN(),           // pow(-Pi, NaN)
	1,               // pow(-1, -Inf) IEEE 754-2008
	1,               // pow(-1, +Inf) IEEE 754-2008
	NaN(),           // pow(-1, NaN)
	Inf(1),          // pow(-1/2, -Inf)
	0,               // pow(-1/2, +Inf)
	Inf(1),          // pow(-0, -Inf)
	Inf(1),          // pow(-0, -Pi)
	Inf(-1),         // pow(-0, -3) IEEE 754-2008
	Copysign(0, -1), // pow(-0, 3) IEEE 754-2008
	0,               // pow(-0, +Pi)
	0,               // pow(-0, +Inf)
	Inf(1),          // pow(+0, -Inf)
	Inf(1),          // pow(+0, -Pi)
	Inf(1),          // pow(+0, -3)
	1,               // pow(+0, -0)
	1,               // pow(+0, +0)
	0,               // pow(+0, 3)
	0,               // pow(+0, +Pi)
	0,               // pow(+0, +Inf)
	NaN(),           // pow(+0, NaN)
	Inf(1),          // pow(1/2, -Inf)
	0,               // pow(1/2, +Inf)
	1,               // pow(1, -Inf) IEEE 754-2008
	1,               // pow(1, +Inf) IEEE 754-2008
	1,               // pow(1, NaN) IEEE 754-2008
	0,               // pow(+Pi, -Inf)
	1,               // pow(+Pi, -0)
	1,               // pow(+Pi, +0)
	Pi,              // pow(+Pi, 1)
	Inf(1),          // pow(+Pi, +Inf)
	NaN(),           // pow(+Pi, NaN)
	0,               // pow(+Inf, -Pi)
	1,               // pow(+Inf, -0)
	1,               // pow(+Inf, +0)
	Inf(1),          // pow(+Inf, 1)
	Inf(1),          // pow(+Inf, Pi)
	NaN(),           // pow(+Inf, NaN)
	NaN(),           // pow(NaN, -Pi)
	1,               // pow(NaN, -0)
	1,               // pow(NaN, +0)
	NaN(),           // pow(NaN, 1)
	NaN(),           // pow(NaN, +Pi)
	NaN(),           // pow(NaN, NaN)
}

var vfsignbitSC = []float32{
	Inf(-1),
	Copysign(0, -1),
	0,
	Inf(1),
	NaN(),
}
var signbitSC = []bool{
	true,
	true,
	false,
	false,
	false,
}

var vfsqrtSC = []float32{
	Inf(-1),
	-Pi,
	Copysign(0, -1),
	0,
	Inf(1),
	NaN(),
	Float32frombits(2), // subnormal; see https://golang.org/issue/13013
}
var sqrtSC = []float32{
	NaN(),
	NaN(),
	Copysign(0, -1),
	0,
	Inf(1),
	NaN(),
	3.1434555694052576e-162,
}

var vftanhSC = []float32{
	Inf(-1),
	Copysign(0, -1),
	0,
	Inf(1),
	NaN(),
}
var tanhSC = []float32{
	-1,
	Copysign(0, -1),
	0,
	1,
	NaN(),
}

// arguments and expected results for boundary cases
const (
	SmallestNormalFloat32   = 1.1754943508222875079687365e-38 // 1/(2**(127-1))
	LargestSubnormalFloat32 = SmallestNormalFloat32 - SmallestNonzeroFloat32
)

var vffrexpBC = []float32{
	SmallestNormalFloat32,
	LargestSubnormalFloat32,
	SmallestNonzeroFloat32,
	MaxFloat32,
	-SmallestNormalFloat32,
	-LargestSubnormalFloat32,
	-SmallestNonzeroFloat32,
	-MaxFloat32,
}
var frexpBC = []fi{
	{0.5, -125},
	{0.9999999, -126},
	{0.5, -148},
	{0.99999994, 128},
	{-0.5, -125},
	{-0.9999999, -126},
	{-0.5, -148},
	{-0.99999994, 128},
}

var vfldexpBC = []fi{
	{SmallestNormalFloat32, -23},
	{LargestSubnormalFloat32, -22},
	{SmallestNonzeroFloat32, 256},
	{MaxFloat32, -(127 + 149)},
	{1, -150},
	{-1, -150},
	{1, 128},
	{-1, 128},
}
var ldexpBC = []float32{
	SmallestNonzeroFloat32,
	3e-45,        // 2**-148
	1.6225928e32, // 2**130
	3e-45,        // 2**-127
	0,
	Copysign(0, -1),
	Inf(1),
	Inf(-1),
}

func tolerance(a, b, e float32) bool {
	d := a - b
	if d < 0 {
		d = -d
	}

	// note: b is correct (expected) value, a is actual value.
	// make error tolerance a fraction of b, not a.
	if b != 0 {
		e = e * b
		if e < 0 {
			e = -e
		}
	}
	return d < e
}
func isClose(a, b float32) bool   { return tolerance(a, b, 1e-5) } // the number gotten from the cfloat standard. Haskell's Linear package uses 1e-6 for floats
func veryclose(a, b float32) bool { return tolerance(a, b, 1e-6) } // from wiki
func alike(a, b float32) bool {
	switch {
	case IsNaN(a) && IsNaN(b):
		return true
	case a == b:
		return Signbit(a) == Signbit(b)
	case a == 0:
		return tolerance(b, a, 1e-22)
	}
	return false
}

func TestNaN(t *testing.T) {
	f32 := NaN()
	if f32 == f32 {
		t.Fatalf("NaN() returns %g, expected NaN", f32)
	}
	f64 := float64(f32)
	if f64 == f64 {
		t.Fatalf("float64(NaN()) is %g, expected NaN", f64)
	}
}

func TestCeil(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Ceil(vf[i]); ceil[i] != f {
			t.Errorf("Ceil(%g) = %g, want %g", vf[i], f, ceil[i])
		}
	}
	for i := 0; i < len(vfceilSC); i++ {
		if f := Ceil(vfceilSC[i]); !alike(ceilSC[i], f) {
			t.Errorf("Ceil(%g) = %g, want %g", vfceilSC[i], f, ceilSC[i])
		}
	}
}

func TestCopysign(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Copysign(vf[i], -1); copysign[i] != f {
			t.Errorf("Copysign(%g, -1) = %g, want %g", vf[i], f, copysign[i])
		}
	}
	for i := 0; i < len(vf); i++ {
		if f := Copysign(vf[i], 1); -copysign[i] != f {
			t.Errorf("Copysign(%g, 1) = %g, want %g", vf[i], f, -copysign[i])
		}
	}
	for i := 0; i < len(vfcopysignSC); i++ {
		if f := Copysign(vfcopysignSC[i], -1); !alike(copysignSC[i], f) {
			t.Errorf("Copysign(%g, -1) = %g, want %g", vfcopysignSC[i], f, copysignSC[i])
		}
	}
}

func TestExp(t *testing.T) {
	testExp(t, Exp, "Exp")
	// testExp(t, ExpGo, "ExpGo")
}

func testExp(t *testing.T, Exp func(float32) float32, name string) {
	for i := 0; i < len(vf); i++ {
		if f := Exp(vf[i]); !isClose(exp[i], f) {
			t.Errorf("%s(%g) = %g, want %g", name, vf[i], f, exp[i])
		}
	}
	for i := 0; i < len(vfexpSC); i++ {
		if f := Exp(vfexpSC[i]); !alike(expSC[i], f) {
			t.Errorf("%s(%g) = %g, want %g", name, vfexpSC[i], f, expSC[i])
		}
	}
}

func TestAbs(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Abs(vf[i]); fabs[i] != f {
			t.Errorf("Abs(%g) = %g, want %g", vf[i], f, fabs[i])
		}
	}
	for i := 0; i < len(vffabsSC); i++ {
		if f := Abs(vffabsSC[i]); !alike(fabsSC[i], f) {
			t.Errorf("Abs(%g) = %g, want %g", vffabsSC[i], f, fabsSC[i])
		}
	}
}

func TestDim(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Dim(vf[i], 0); fdim[i] != f {
			t.Errorf("Dim(%g, %g) = %g, want %g", vf[i], 0.0, f, fdim[i])
		}
	}
	for i := 0; i < len(vffdimSC); i++ {
		if f := Dim(vffdimSC[i][0], vffdimSC[i][1]); !alike(fdimSC[i], f) {
			t.Errorf("Dim(%g, %g) = %g, want %g", vffdimSC[i][0], vffdimSC[i][1], f, fdimSC[i])
		}
	}
	for i := 0; i < len(vffdim2SC); i++ {
		if f := Dim(vffdim2SC[i][0], vffdim2SC[i][1]); !alike(fdimSC[i], f) {
			t.Errorf("Dim(%g, %g) = %g, want %g", vffdim2SC[i][0], vffdim2SC[i][1], f, fdimSC[i])
		}
	}
}

func TestFloor(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Floor(vf[i]); floor[i] != f {
			t.Errorf("Floor(%g) = %g, want %g", vf[i], f, floor[i])
		}
	}
	for i := 0; i < len(vfceilSC); i++ {
		if f := Floor(vfceilSC[i]); !alike(ceilSC[i], f) {
			t.Errorf("Floor(%g) = %g, want %g", vfceilSC[i], f, ceilSC[i])
		}
	}
}

func TestMax(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Max(vf[i], ceil[i]); ceil[i] != f {
			t.Errorf("Max(%g, %g) = %g, want %g", vf[i], ceil[i], f, ceil[i])
		}
	}
	for i := 0; i < len(vffdimSC); i++ {
		if f := Max(vffdimSC[i][0], vffdimSC[i][1]); !alike(fmaxSC[i], f) {
			t.Errorf("Max(%g, %g) = %g, want %g", vffdimSC[i][0], vffdimSC[i][1], f, fmaxSC[i])
		}
	}
	for i := 0; i < len(vffdim2SC); i++ {
		if f := Max(vffdim2SC[i][0], vffdim2SC[i][1]); !alike(fmaxSC[i], f) {
			t.Errorf("Max(%g, %g) = %g, want %g", vffdim2SC[i][0], vffdim2SC[i][1], f, fmaxSC[i])
		}
	}
}

func TestMin(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Min(vf[i], floor[i]); floor[i] != f {
			t.Errorf("Min(%g, %g) = %g, want %g", vf[i], floor[i], f, floor[i])
		}
	}
	for i := 0; i < len(vffdimSC); i++ {
		if f := Min(vffdimSC[i][0], vffdimSC[i][1]); !alike(fminSC[i], f) {
			t.Errorf("Min(%g, %g) = %g, want %g", vffdimSC[i][0], vffdimSC[i][1], f, fminSC[i])
		}
	}
	for i := 0; i < len(vffdim2SC); i++ {
		if f := Min(vffdim2SC[i][0], vffdim2SC[i][1]); !alike(fminSC[i], f) {
			t.Errorf("Min(%g, %g) = %g, want %g", vffdim2SC[i][0], vffdim2SC[i][1], f, fminSC[i])
		}
	}
}

func TestFrexp(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f, j := Frexp(vf[i]); !veryclose(frexp[i].f, f) || frexp[i].i != j {
			t.Errorf("Frexp(%g) = %g, %d, want %g, %d", vf[i], f, j, frexp[i].f, frexp[i].i)
		}
	}
	for i := 0; i < len(vffrexpSC); i++ {
		if f, j := Frexp(vffrexpSC[i]); !alike(frexpSC[i].f, f) || frexpSC[i].i != j {
			t.Errorf("Frexp(%g) = %g, %d, want %g, %d", vffrexpSC[i], f, j, frexpSC[i].f, frexpSC[i].i)
		}
	}
	for i := 0; i < len(vffrexpBC); i++ {
		if f, j := Frexp(vffrexpBC[i]); !alike(frexpBC[i].f, f) || frexpBC[i].i != j {
			t.Errorf("Frexp(%g) = %g, %d, want %g, %d", vffrexpBC[i], f, j, frexpBC[i].f, frexpBC[i].i)
		}
	}
}

func TestLdexp(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Ldexp(frexp[i].f, frexp[i].i); !veryclose(vf[i], f) {
			t.Errorf("Ldexp(%g, %d) = %g, want %g", frexp[i].f, frexp[i].i, f, vf[i])
		}
	}
	for i := 0; i < len(vffrexpSC); i++ {
		if f := Ldexp(frexpSC[i].f, frexpSC[i].i); !alike(vffrexpSC[i], f) {
			t.Errorf("Ldexp(%g, %d) = %g, want %g", frexpSC[i].f, frexpSC[i].i, f, vffrexpSC[i])
		}
	}
	for i := 0; i < len(vfldexpSC); i++ {
		if f := Ldexp(vfldexpSC[i].f, vfldexpSC[i].i); !alike(ldexpSC[i], f) {
			t.Errorf("Ldexp(%g, %d) = %g, want %g", vfldexpSC[i].f, vfldexpSC[i].i, f, ldexpSC[i])
		}
	}
	for i := 0; i < len(vffrexpBC); i++ {
		if f := Ldexp(frexpBC[i].f, frexpBC[i].i); !alike(vffrexpBC[i], f) {
			t.Errorf("Ldexp(%g, %d) = %g, want %g", frexpBC[i].f, frexpBC[i].i, f, vffrexpBC[i])
		}
	}
	for i := 0; i < len(vfldexpBC); i++ {
		if f := Ldexp(vfldexpBC[i].f, vfldexpBC[i].i); !alike(ldexpBC[i], f) {
			t.Errorf("Ldexp(%g, %d) = %g, want %g", vfldexpBC[i].f, vfldexpBC[i].i, f, ldexpBC[i])
		}
	}
}

func TestLog(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		a := Abs(vf[i])
		if f := Log(a); !isClose(log[i], f) {
			t.Errorf("Log(%g) = %g, want %g", a, f, log[i])
		}
	}
	if f := Log(10); f != Ln10 {
		t.Errorf("Log(%g) = %g, want %g", 10.0, f, Ln10)
	}
	for i := 0; i < len(vflogSC); i++ {
		if f := Log(vflogSC[i]); !alike(logSC[i], f) {
			t.Errorf("Log(%g) = %g, want %g", vflogSC[i], f, logSC[i])
		}
	}
}

func TestModf(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f, g := Modf(vf[i]); !veryclose(modf[i][0], f) || !tolerance(modf[i][1], g, 1e-4) {
			t.Errorf("Modf(%g) = %g, %g, want %g, %g", vf[i], f, g, modf[i][0], modf[i][1])
		}
	}
	for i := 0; i < len(vfmodfSC); i++ {
		if f, g := Modf(vfmodfSC[i]); !alike(modfSC[i][0], f) || !alike(modfSC[i][1], g) {
			t.Errorf("Modf(%g) = %g, %g, want %g, %g", vfmodfSC[i], f, g, modfSC[i][0], modfSC[i][1])
		}
	}
}

func TestPow(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Pow(10, vf[i]); !isClose(pow[i], f) {
			t.Errorf("Pow(10, %g) = %g, want %g", vf[i], f, pow[i])
		}
	}
	for i := 0; i < len(vfpowSC); i++ {
		if f := Pow(vfpowSC[i][0], vfpowSC[i][1]); !alike(powSC[i], f) {
			t.Errorf("Pow(%g, %g) = %g, want %g", vfpowSC[i][0], vfpowSC[i][1], f, powSC[i])
		}
	}
}

func TestSignbit(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Signbit(vf[i]); signbit[i] != f {
			t.Errorf("Signbit(%g) = %t, want %t", vf[i], f, signbit[i])
		}
	}
	for i := 0; i < len(vfsignbitSC); i++ {
		if f := Signbit(vfsignbitSC[i]); signbitSC[i] != f {
			t.Errorf("Signbit(%g) = %t, want %t", vfsignbitSC[i], f, signbitSC[i])
		}
	}
}

func TestSqrt(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		a := Abs(vf[i])
		if f := SqrtGo(a); !veryclose(sqrt[i], f) {
			t.Errorf("SqrtGo(%g) = %g, want %g", a, f, sqrt[i])
		}
		a = Abs(vf[i])
		if f := Sqrt(a); !veryclose(sqrt[i], f) {
			t.Errorf("Sqrt(%g) = %g, want %g", a, f, sqrt[i])
		}
	}
	for i := 0; i < len(vfsqrtSC); i++ {
		if f := SqrtGo(vfsqrtSC[i]); !alike(sqrtSC[i], f) {
			t.Errorf("SqrtGo(%g) = %g, want %g", vfsqrtSC[i], f, sqrtSC[i])
		}
		if f := Sqrt(vfsqrtSC[i]); !alike(sqrtSC[i], f) {
			t.Errorf("Sqrt(%g) = %g, want %g", vfsqrtSC[i], f, sqrtSC[i])
		}
	}
}

func TestTanh(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Tanh(vf[i]); !veryclose(tanh[i], f) {
			t.Errorf("Tanh(%g) = %g, want %g", vf[i], f, tanh[i])
		}
	}
	for i := 0; i < len(vftanhSC); i++ {
		if f := Tanh(vftanhSC[i]); !alike(tanhSC[i], f) {
			t.Errorf("Tanh(%g) = %g, want %g", vftanhSC[i], f, tanhSC[i])
		}
	}
}

func TestTrunc(t *testing.T) {
	for i := 0; i < len(vf); i++ {
		if f := Trunc(vf[i]); trunc[i] != f {
			t.Errorf("Trunc(%g) = %g, want %g", vf[i], f, trunc[i])
		}
	}
	for i := 0; i < len(vfceilSC); i++ {
		if f := Trunc(vfceilSC[i]); !alike(ceilSC[i], f) {
			t.Errorf("Trunc(%g) = %g, want %g", vfceilSC[i], f, ceilSC[i])
		}
	}
}

// Check that math constants are accepted by compiler
// and have right value (assumes strconv.ParseFloat works).
// https://golang.org/issue/201

type floatTest struct {
	val  interface{}
	name string
	str  string
}

var floatTests = []floatTest{
	{float32(MaxFloat32), "MaxFloat32", "3.4028235e+38"},
	{float32(SmallestNonzeroFloat32), "SmallestNonzeroFloat32", "1e-45"},
}

func TestFloatMinMax(t *testing.T) {
	for _, tt := range floatTests {
		s := fmt.Sprint(tt.val)
		if s != tt.str {
			t.Errorf("Sprint(%v) = %s, want %s", tt.name, s, tt.str)
		}
	}
}

// Benchmarks

func BenchmarkCeil(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Ceil(.5)
	}
}

func BenchmarkCopysign(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Copysign(.5, -1)
	}
}

func BenchmarkExp(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Exp(.5)
	}
}

func BenchmarkAbs(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Abs(.5)
	}
}

func BenchmarkDim(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Dim(10, 3)
	}
}

func BenchmarkFloor(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Floor(.5)
	}
}

func BenchmarkMax(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Max(10, 3)
	}
}

func BenchmarkMin(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Min(10, 3)
	}
}

func BenchmarkFrexp(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Frexp(8)
	}
}

func BenchmarkLdexp(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Ldexp(.5, 2)
	}
}

func BenchmarkLog(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Log(.5)
	}
}

func BenchmarkModf(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Modf(1.5)
	}
}

func BenchmarkPowInt(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Pow(2, 2)
	}
}

func BenchmarkPowFrac(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Pow(2.5, 1.5)
	}
}

func BenchmarkSignbit(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Signbit(2.5)
	}
}

func BenchmarkSqrt(b *testing.B) {
	var x, y float32
	x, y = 0.0, 10.0
	for i := 0; i < b.N; i++ {
		x += Sqrt(y)
	}
}

func BenchmarkSqrtIndirect(b *testing.B) {
	var x, y float32
	x, y = 0.0, 10.0
	f := Sqrt
	for i := 0; i < b.N; i++ {
		x += f(y)
	}
}

func isPrime(i int) bool {
	// Yes, this is a dumb way to write this code,
	// but calling Sqrt repeatedly in this way demonstrates
	// the benefit of using a direct SQRT instruction on systems
	// that have one, whereas the obvious loop seems not to
	// demonstrate such a benefit.
	for j := 2; float32(j) <= Sqrt(float32(i)); j++ {
		if i%j == 0 {
			return false
		}
	}
	return true
}

func BenchmarkSqrtPrime(b *testing.B) {
	for i := 0; i < b.N; i++ {
		isPrime(100003)
	}
}

func BenchmarkTanh(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Tanh(2.5)
	}
}
func BenchmarkTrunc(b *testing.B) {
	for i := 0; i < b.N; i++ {
		Trunc(.5)
	}
}


<|start_filename|>pkg/nlp/gbst/utils.go<|end_filename|>
package gbst

import "math"

// greatest common divisor (GCD) via Euclidean algorithm
func gcd(a, b int) int {
	for b != 0 {
		a, b = b, a%b
	}
	return a
}

// lcm finds the Least Common Multiple (LCM) via GCD
func lcm(a, b int, integers ...int) int {
	result := a * b / gcd(a, b)
	for i := 0; i < len(integers); i++ {
		result = lcm(result, integers[i])
	}
	return result
}

// nextDivisibleLength finds the greater value next to sequence length value, divisible by multiple
func nextDivisibleLength(seqLen, multiple int) int {
	return int(math.Ceil(float64(seqLen)/float64(multiple))) * multiple
}


<|start_filename|>pkg/ml/nn/normalization/batchnorm/batchnorm.go<|end_filename|>
// Copyright 2019 spaGO Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package batchnorm

import (
	"encoding/gob"
	mat "github.com/nlpodyssey/spago/pkg/mat32"
	"github.com/nlpodyssey/spago/pkg/ml/ag"
	"github.com/nlpodyssey/spago/pkg/ml/nn"
)

var (
	_ nn.Model = &Model{}
)

// Model contains the serializable parameters.
type Model struct {
	nn.BaseModel
	W        nn.Param `spago:"type:weights"`
	B        nn.Param `spago:"type:biases"`
	Mean     nn.Param `spago:"type:undefined"`
	StdDev   nn.Param `spago:"type:undefined"`
	Momentum nn.Param `spago:"type:undefined"`
}

const epsilon = 1e-5
const defaultMomentum = 0.9

func init() {
	gob.Register(&Model{})
}

// NewWithMomentum returns a new model with supplied size and momentum.
func NewWithMomentum(size int, momentum mat.Float) *Model {
	return &Model{
		W:        nn.NewParam(mat.NewInitVecDense(size, epsilon)),
		B:        nn.NewParam(mat.NewEmptyVecDense(size)),
		Mean:     nn.NewParam(mat.NewEmptyVecDense(size), nn.RequiresGrad(false)),
		StdDev:   nn.NewParam(mat.NewEmptyVecDense(size), nn.RequiresGrad(false)),
		Momentum: nn.NewParam(mat.NewScalar(momentum), nn.RequiresGrad(false)),
	}
}

// New returns a new model with the supplied size and default momentum
func New(size int) *Model {
	return NewWithMomentum(size, defaultMomentum)
}

// Forward performs the forward step for each input node and returns the result.
func (m *Model) Forward(xs ...ag.Node) []ag.Node {
	if m.Mode() == nn.Training {
		return m.forwardTraining(xs)
	}
	return m.forwardInference(xs)
}

func (m *Model) forwardTraining(xs []ag.Node) []ag.Node {
	g := m.Graph()
	meanVector := m.mean(xs)
	devVector := m.stdDev(meanVector, xs)
	m.updateBatchNormParameters(meanVector.Value(), devVector.Value())
	return m.process(g, xs, devVector, meanVector)
}

func (m *Model) process(g *ag.Graph, xs []ag.Node, devVector ag.Node, meanVector ag.Node) []ag.Node {
	devVector = g.Div(m.W, g.AddScalar(devVector, g.NewScalar(epsilon)))
	ys := make([]ag.Node, len(xs))
	for i, x := range xs {
		ys[i] = g.Add(g.Prod(g.Sub(x, meanVector), devVector), m.B)
	}
	return ys
}

func (m *Model) updateBatchNormParameters(meanVector, devVector mat.Matrix) {
	momentum := m.Momentum.Value().Scalar()

	m.Mean.ReplaceValue(
		m.Mean.Value().ProdScalar(momentum).Add(meanVector.ProdScalar(1.0 - momentum)))

	m.StdDev.ReplaceValue(
		m.StdDev.Value().ProdScalar(momentum).Add(devVector.ProdScalar(1.0 - momentum)))
}

func (m *Model) forwardInference(xs []ag.Node) []ag.Node {
	g := m.Graph()
	meanVector := g.NewWrapNoGrad(m.Mean)
	devVector := g.NewWrapNoGrad(m.StdDev)
	return m.process(g, xs, devVector, meanVector)
}

// Mean computes the mean of the input.
func (m *Model) mean(xs []ag.Node) ag.Node {
	g := m.Graph()
	sumVector := xs[0]
	for i := 1; i < len(xs); i++ {
		sumVector = g.Add(sumVector, xs[i])
	}

	return g.DivScalar(sumVector, g.NewScalar(mat.Float(len(xs))+epsilon))
}

// StdDev computes the standard deviation of the input.
func (m *Model) stdDev(meanVector ag.Node, xs []ag.Node) ag.Node {
	g := m.Graph()
	devVector := g.NewVariable(meanVector.Value().ZerosLike(), false)
	for _, x := range xs {
		diffVector := g.Square(g.Sub(meanVector, x))
		devVector = g.Add(devVector, diffVector)
	}
	devVector = g.Sqrt(g.DivScalar(devVector, g.NewScalar(mat.Float(len(xs))+epsilon)))
	return devVector
}


<|start_filename|>pkg/mat32/internal/math32/export_test.go<|end_filename|>
package math32

// Export internal functions for testing.

var SqrtGo = sqrt
