Sarthak
commited on
Commit
·
53a6528
1
Parent(s):
bba24e6
feat(config): add new model type for vector embeddings
Browse filesAdd a new model type, StaticModel, for vector embeddings.
This commit introduces a new model type, StaticModel, to enhance the capability of creating vector embeddings. This is achieved by updating the configuration file and tokenizer configuration to include the definition of the new model type.
- config.json +10 -0
- tokenizer.json +2 -2
config.json
CHANGED
@@ -1,3 +1,13 @@
|
|
1 |
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2 |
"normalize": true
|
3 |
}
|
|
|
1 |
{
|
2 |
+
"model_type": "model2vec",
|
3 |
+
"architectures": [
|
4 |
+
"StaticModel"
|
5 |
+
],
|
6 |
+
"tokenizer_name": "sentence-transformers/all-mpnet-base-v2",
|
7 |
+
"apply_pca": 256,
|
8 |
+
"apply_zipf": true,
|
9 |
+
"sif_coefficient": 0.001,
|
10 |
+
"hidden_dim": 256,
|
11 |
+
"seq_length": 1000000,
|
12 |
"normalize": true
|
13 |
}
|
tokenizer.json
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:204cce1d589a5a74f3507122a40a87b4405fc17af12e070c2be8851631ad6b6a
|
3 |
+
size 682351
|