initial commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

ModernBERT-base.mlpackage/Data/com.apple.CoreML/FeatureDescriptions.json

@@ -0,0 +1,21 @@
+{
+  "Inputs" : {
+    "attention_mask" : {
+      "MLFeatureShortDescription" : "--"
+    },
+    "input_ids" : {
+      "MLFeatureShortDescription" : "--"
+    }
+  },
+  "Outputs" : {
+    "logits" : {
+      "MLFeatureShortDescription" : "--"
+    }
+  },
+  "TrainingInputs" : {
+
+  },
+  "States" : {
+
+  }
+}

ModernBERT-base.mlpackage/Data/com.apple.CoreML/Metadata.json

@@ -0,0 +1,11 @@
+{
+  "MLModelVersionStringKey" : "--",
+  "MLModelDescriptionKey" : "ModernBERT is a modernized bidirectional encoder-only Transformer model (BERT-style) pre-trained on 2 trillion tokens of English and code data with a native context length of up to 8,192 tokens.\n\nhttps:\/\/huggingface.co\/answerdotai\/ModernBERT-base",
+  "MLModelCreatorDefinedKey" : {
+    "com.github.apple.coremltools.source_dialect" : "TorchScript",
+    "com.github.apple.coremltools.source" : "torch==2.5.1",
+    "com.github.apple.coremltools.version" : "8.1"
+  },
+  "MLModelAuthorKey" : "Answer.AI",
+  "MLModelLicenseKey" : "Apache license 2.0"
+}

ModernBERT-base.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebcc912b77e91056515709e659752d132ad97e78e2e4a6032bee1aec80fedf90
+size 266649

ModernBERT-base.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4e6706d003041ba9244f96433429f7bf9c9de12ad17df917b947dc691b581e3
+size 299325824

ModernBERT-base.mlpackage/Manifest.json

@@ -0,0 +1,30 @@
+{
+    "fileFormatVersion": "1.0.0",
+    "itemInfoEntries": {
+        "06E6264F-CB31-4D71-A23C-A3D3689A565C": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Weights",
+            "name": "weights",
+            "path": "com.apple.CoreML/weights"
+        },
+        "5510B1F0-2CBA-49A0-B455-0C475F598606": {
+            "author": "com.apple.CoreML",
+            "description": "External FeatureDescription Overlay",
+            "name": "FeatureDescriptions.json",
+            "path": "com.apple.CoreML/FeatureDescriptions.json"
+        },
+        "90F77D61-5B31-4442-BA5D-CFF0227A04B1": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Specification",
+            "name": "model.mlmodel",
+            "path": "com.apple.CoreML/model.mlmodel"
+        },
+        "F4E8C7BD-D5AA-4A9E-B724-C70CE42D4C48": {
+            "author": "com.apple.CoreML",
+            "description": "External Metadata Overlay",
+            "name": "Metadata.json",
+            "path": "com.apple.CoreML/Metadata.json"
+        }
+    },
+    "rootModelIdentifier": "90F77D61-5B31-4442-BA5D-CFF0227A04B1"
+}

ModernBERT-large.mlpackage/Data/com.apple.CoreML/FeatureDescriptions.json

@@ -0,0 +1,21 @@
+{
+  "Inputs" : {
+    "attention_mask" : {
+      "MLFeatureShortDescription" : ""
+    },
+    "input_ids" : {
+      "MLFeatureShortDescription" : ""
+    }
+  },
+  "Outputs" : {
+    "logits" : {
+      "MLFeatureShortDescription" : ""
+    }
+  },
+  "TrainingInputs" : {
+
+  },
+  "States" : {
+
+  }
+}

ModernBERT-large.mlpackage/Data/com.apple.CoreML/Metadata.json

@@ -0,0 +1,10 @@
+{
+  "MLModelDescriptionKey" : "ModernBERT is a modernized bidirectional encoder-only Transformer model (BERT-style) pre-trained on 2 trillion tokens of English and code data with a native context length of up to 8,192 tokens.\n\nhttps:\/\/huggingface.co\/answerdotai\/ModernBERT-large",
+  "MLModelCreatorDefinedKey" : {
+    "com.github.apple.coremltools.source_dialect" : "TorchScript",
+    "com.github.apple.coremltools.version" : "8.1",
+    "com.github.apple.coremltools.source" : "torch==2.5.1"
+  },
+  "MLModelAuthorKey" : "Answer.AI",
+  "MLModelLicenseKey" : "Apache license 2.0"
+}

ModernBERT-large.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cf60aa2e976e2c594e83db8f8ba42ecf9cc9f14348b4221347b7df6f612a8bd
+size 335419

ModernBERT-large.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90b125e16fb2050ae1c29c076c2733d167e556bdd853e9e917c120eeb87a7c25
+size 791793600

ModernBERT-large.mlpackage/Manifest.json

@@ -0,0 +1,30 @@
+{
+    "fileFormatVersion": "1.0.0",
+    "itemInfoEntries": {
+        "0B8AE6F6-1317-4127-8BD5-2AB46EC557CE": {
+            "author": "com.apple.CoreML",
+            "description": "External Metadata Overlay",
+            "name": "Metadata.json",
+            "path": "com.apple.CoreML/Metadata.json"
+        },
+        "3D71FBFB-60F3-48A0-8F1B-8550A02403E1": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Specification",
+            "name": "model.mlmodel",
+            "path": "com.apple.CoreML/model.mlmodel"
+        },
+        "56305CCC-0DD8-43A2-8834-124ACA88424A": {
+            "author": "com.apple.CoreML",
+            "description": "External FeatureDescription Overlay",
+            "name": "FeatureDescriptions.json",
+            "path": "com.apple.CoreML/FeatureDescriptions.json"
+        },
+        "75F45B20-DDAB-400A-8F8E-87D2E0A4DFB6": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Weights",
+            "name": "weights",
+            "path": "com.apple.CoreML/weights"
+        }
+    },
+    "rootModelIdentifier": "3D71FBFB-60F3-48A0-8F1B-8550A02403E1"
+}

README.md

@@ -0,0 +1,48 @@
+---
+license: apache-2.0
+language:
+- en
+tags:
+- fill-mask
+- masked-lm
+- long-context
+- modernbert
+---
+
+# ModernBERT-CoreML
+
+This repo contains [ModernBERT-base](https://huggingface.co/answerdotai/ModernBERT-base) and [ModernBERT-large](https://huggingface.co/answerdotai/ModernBERT-large) converted to CoreML.
+
+### Example Usage
+
+```swift
+import CoreML
+import Tokenizers
+
+let text = "The capital of Ireland is [MASK]."
+
+print("Loading…")
+let model = try await ModernBERT_base.load()
+let tokenizer = try await AutoTokenizer.from(pretrained: "answerdotai/ModernBERT-base")
+
+let tokens = tokenizer(text)
+let inputIDs = MLShapedArray(scalars: tokens.map(Int32.init), shape: [1, tokens.count])
+let attentionMask = MLShapedArray<Int32>(scalars: tokens.map { _ in 1 }, shape: [1, tokens.count])
+let input = ModernBERT_baseInput(
+    input_ids: inputIDs,
+    attention_mask: attentionMask
+)
+
+print("Predicting…")
+let output = try await model.prediction(input: input)
+let logits = output.logitsShapedArray
+let maskPosition = tokens.firstIndex(
+    of: tokenizer.encode(text: "[MASK]", addSpecialTokens: false)[0]
+)!
+let maskLogits = logits[0, maskPosition]
+let maskToken = Int(await MLTensor(maskLogits).argmax().shapedArray(of: Int32.self).scalar!)
+let maskText = tokenizer.decode(tokens: [maskToken])
+
+// The capital of Ireland is Dublin.
+print(text.replacingOccurrences(of: "[MASK]", with: maskText.trimmingCharacters(in: .whitespaces)))
+```