first model version

README.md

@@ -1,3 +1,55 @@
 ---
-license: cc
 ---
+
+# swiss\_german\_pos\_model
+
+The *swiss_german_pos_model* is a part-of-speech tagging model for Swiss German. The model is trained on [Universal POS tags (upos)](https://universaldependencies.org/u/pos/).
+
+### Training procedure and data sets
+
+- Base model: German LM: [dbmdz/bert-base-german-cased](https://huggingface.co/dbmdz/bert-base-german-cased)
+- Continued LM training with [swisscrawl data](https://icosys.ch/swisscrawl)
+- Task fine-tuning on the [UD\_German-HDT](https://github.com/UniversalDependencies/UD_German-HDT/tree/master) data set with [character-level noise](https://aclanthology.org/2022.findings-acl.321/)
+- Task fine-tuning on the Swiss German [NOAH-Corpus](https://noe-eva.github.io/NOAH-Corpus/) (train + dev split)
+
+Accuracy on NOAH test split: 0.9587
+
+
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 1
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- num_epochs: 5.0
+
+### Framework versions
+
+- Transformers 4.25.0.dev0
+- Pytorch 1.13.1
+- Datasets 2.8.0
+- Tokenizers 0.13.2
+
+
+### Citation
+
+``` @inproceedings{aepli-sennrich-2022-improving,
+    title = "Improving Zero-Shot Cross-lingual Transfer Between Closely Related Languages by Injecting Character-Level Noise",
+    author = {Aepli, No{\"e}mi  and
+      Sennrich, Rico},
+    booktitle = "Findings of the Association for Computational Linguistics: ACL 2022",
+    month = may,
+    year = "2022",
+    address = "Dublin, Ireland",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2022.findings-acl.321",
+    doi = "10.18653/v1/2022.findings-acl.321",
+    pages = "4074--4083",
+    abstract = "Cross-lingual transfer between a high-resource language and its dialects or closely related language varieties should be facilitated by their similarity. However, current approaches that operate in the embedding space do not take surface similarity into account. This work presents a simple yet effective strategy to improve cross-lingual transfer between closely related varieties. We propose to augment the data of the high-resource source language with character-level noise to make the model more robust towards spelling variations. Our strategy shows consistent improvements over several languages and tasks: Zero-shot transfer of POS tagging and topic identification between language varieties from the Finnic, West and North Germanic, and Western Romance language branches. Our work provides evidence for the usefulness of simple surface-level noise in improving transfer between language varieties.",
+} ```
+
+

config.json

@@ -0,0 +1,63 @@
+{
+  "_name_or_path": "/home/user/naepli/noisepp/gsw_pos_best/DEbert_swisscrawl_ftDE-noise",
+  "architectures": [
+    "BertForTokenClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "finetuning_task": "ner",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "ADJ",
+    "1": "ADP",
+    "2": "ADV",
+    "3": "AUX",
+    "4": "CCONJ",
+    "5": "DET",
+    "6": "INTJ",
+    "7": "NOUN",
+    "8": "NUM",
+    "9": "PART",
+    "10": "PRON",
+    "11": "PROPN",
+    "12": "PUNCT",
+    "13": "SCONJ",
+    "14": "VERB",
+    "15": "X"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "ADJ": 0,
+    "ADP": 1,
+    "ADV": 2,
+    "AUX": 3,
+    "CCONJ": 4,
+    "DET": 5,
+    "INTJ": 6,
+    "NOUN": 7,
+    "NUM": 8,
+    "PART": 9,
+    "PRON": 10,
+    "PROPN": 11,
+    "PUNCT": 12,
+    "SCONJ": 13,
+    "VERB": 14,
+    "X": 15
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.25.0.dev0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 31102
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62da48eb0d63620e2ee9154e6cb1d244298016bea8ba3ba7f2722cb8a5afaffa
+size 437469741

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,16 @@
+{
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "mask_token": "[MASK]",
+  "max_len": 512,
+  "name_or_path": "/home/user/naepli/noisepp/gsw_pos_best/DEbert_swisscrawl_ftDE-noise",
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

trainer_state.json

@@ -0,0 +1,97 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "global_step": 3205,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.78,
+      "learning_rate": 4.219968798751951e-05,
+      "loss": 0.2448,
+      "step": 500
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 3.4399375975039005e-05,
+      "loss": 0.117,
+      "step": 1000
+    },
+    {
+      "epoch": 1.56,
+      "eval_accuracy": 0.9510651573223995,
+      "eval_f1": 0.9413793103448275,
+      "eval_loss": 0.17917607724666595,
+      "eval_precision": 0.942101226993865,
+      "eval_recall": 0.9406584992343032,
+      "eval_runtime": 1.3355,
+      "eval_samples_per_second": 548.105,
+      "eval_steps_per_second": 68.887,
+      "step": 1000
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 2.65990639625585e-05,
+      "loss": 0.0721,
+      "step": 1500
+    },
+    {
+      "epoch": 3.12,
+      "learning_rate": 1.8798751950078e-05,
+      "loss": 0.045,
+      "step": 2000
+    },
+    {
+      "epoch": 3.12,
+      "eval_accuracy": 0.9569480345841875,
+      "eval_f1": 0.9488834696122029,
+      "eval_loss": 0.20152758061885834,
+      "eval_precision": 0.948112756808409,
+      "eval_recall": 0.9496554364471669,
+      "eval_runtime": 1.2725,
+      "eval_samples_per_second": 575.238,
+      "eval_steps_per_second": 72.298,
+      "step": 2000
+    },
+    {
+      "epoch": 3.9,
+      "learning_rate": 1.0998439937597505e-05,
+      "loss": 0.0252,
+      "step": 2500
+    },
+    {
+      "epoch": 4.68,
+      "learning_rate": 3.198127925117005e-06,
+      "loss": 0.0142,
+      "step": 3000
+    },
+    {
+      "epoch": 4.68,
+      "eval_accuracy": 0.957215438096087,
+      "eval_f1": 0.94943793350873,
+      "eval_loss": 0.22939395904541016,
+      "eval_precision": 0.9490293583245673,
+      "eval_recall": 0.9498468606431854,
+      "eval_runtime": 1.2703,
+      "eval_samples_per_second": 576.223,
+      "eval_steps_per_second": 72.421,
+      "step": 3000
+    },
+    {
+      "epoch": 5.0,
+      "step": 3205,
+      "total_flos": 824467030346496.0,
+      "train_loss": 0.0817642333912961,
+      "train_runtime": 184.8416,
+      "train_samples_per_second": 138.605,
+      "train_steps_per_second": 17.339
+    }
+  ],
+  "max_steps": 3205,
+  "num_train_epochs": 5,
+  "total_flos": 824467030346496.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fec32aecbf7fe52f78338d4fb3a3ffe068a6b8604bc7ef398d124d9931953d4
+size 3515