Ankur Goyal commited on
Commit
73a21c8
1 Parent(s): 57b0bdd

Initial commit

Browse files
README.md CHANGED
@@ -1,3 +1,15 @@
1
  ---
 
 
2
  license: mit
3
  ---
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ language: en
3
+ thumbnail: https://uploads-ssl.webflow.com/5e3898dff507782a6580d710/614a23fcd8d4f7434c765ab9_logo.png
4
  license: mit
5
  ---
6
+
7
+ # LayoutLM for Visual Question Answering
8
+
9
+ This is a fine-tuned version of the multi-modal [LayoutLM](https://aka.ms/layoutlm) model for the task of question answering on documents. It has been fine-tuned on
10
+
11
+ ## Model details
12
+
13
+ The LayoutLM model was developed at Microsoft ([paper](https://arxiv.org/abs/1912.13318)) as a general purpose tool for understanding documents. This model is a fine-tuned checkpoint of [LayoutLM-Base-Cased](https://huggingface.co/microsoft/layoutlm-base-uncased), using both the [SQuAD2.0](https://huggingface.co/datasets/squad_v2) and [DocVQA](https://www.docvqa.org/) datasets.
14
+
15
+ ## Getting started with the model
config.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "attention_probs_dropout_prob": 0.1,
3
+ "architectures": [
4
+ "LayoutLMForQuestionAnswering"
5
+ ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_layoutlm.LayoutLMConfig",
8
+ "AutoModelForQuestionAnswering": "modeling_layoutlm.LayoutLMForQuestionAnswering"
9
+ },
10
+ "custom_pipelines": {
11
+ "document-question-answering": {
12
+ "impl": "pipeline_document_question_answering.DocumentQuestionAnsweringPipeline",
13
+ "pt": "AutoModelForQuestionAnswering"
14
+ }
15
+ },
16
+ "bos_token_id": 0,
17
+ "eos_token_id": 2,
18
+ "gradient_checkpointing": false,
19
+ "hidden_act": "gelu",
20
+ "hidden_dropout_prob": 0.1,
21
+ "hidden_size": 768,
22
+ "initializer_range": 0.02,
23
+ "intermediate_size": 3072,
24
+ "layer_norm_eps": 1e-05,
25
+ "max_2d_position_embeddings": 1024,
26
+ "max_position_embeddings": 514,
27
+ "model_type": "layoutlm",
28
+ "num_attention_heads": 12,
29
+ "num_hidden_layers": 12,
30
+ "pad_token_id": 1,
31
+ "position_embedding_type": "absolute",
32
+ "tokenizer_class": "RobertaTokenizer",
33
+ "transformers_version": "4.6.1",
34
+ "type_vocab_size": 1,
35
+ "use_cache": true,
36
+ "vocab_size": 50265
37
+ }
configuration_layoutlm.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # This model just uses the existing LayoutLMConfig which is just imported
2
+ # as a thin wrapper
3
+ from transformers.models.layoutlm.configuration_layoutlm import LayoutLMConfig
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
modeling_layoutlm.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # NOTE: This code is currently under review for inclusion in the main
2
+ # huggingface/transformers repository:
3
+ # https://github.com/huggingface/transformers/pull/18407
4
+ """ PyTorch LayoutLM model."""
5
+
6
+
7
+ import math
8
+ from typing import Optional, Tuple, Union
9
+
10
+ import torch
11
+ from torch import nn
12
+ from torch.nn import CrossEntropyLoss
13
+
14
+ from transformers.modeling_outputs import QuestionAnsweringModelOutput
15
+ from transformers.models.layoutlm import LayoutLMModel, LayoutLMPreTrainedModel
16
+
17
+
18
+ class LayoutLMForQuestionAnswering(LayoutLMPreTrainedModel):
19
+ def __init__(self, config, has_visual_segment_embedding=True):
20
+ super().__init__(config)
21
+ self.num_labels = config.num_labels
22
+
23
+ self.layoutlm = LayoutLMModel(config)
24
+ self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels)
25
+
26
+ # Initialize weights and apply final processing
27
+ self.post_init()
28
+
29
+ def get_input_embeddings(self):
30
+ return self.layoutlm.embeddings.word_embeddings
31
+
32
+ def forward(
33
+ self,
34
+ input_ids: Optional[torch.LongTensor] = None,
35
+ bbox: Optional[torch.LongTensor] = None,
36
+ attention_mask: Optional[torch.FloatTensor] = None,
37
+ token_type_ids: Optional[torch.LongTensor] = None,
38
+ position_ids: Optional[torch.LongTensor] = None,
39
+ head_mask: Optional[torch.FloatTensor] = None,
40
+ inputs_embeds: Optional[torch.FloatTensor] = None,
41
+ start_positions: Optional[torch.LongTensor] = None,
42
+ end_positions: Optional[torch.LongTensor] = None,
43
+ output_attentions: Optional[bool] = None,
44
+ output_hidden_states: Optional[bool] = None,
45
+ return_dict: Optional[bool] = None,
46
+ ) -> Union[Tuple, QuestionAnsweringModelOutput]:
47
+ r"""
48
+ start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
49
+ Labels for position (index) of the start of the labelled span for computing the token classification loss.
50
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
51
+ are not taken into account for computing the loss.
52
+ end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
53
+ Labels for position (index) of the end of the labelled span for computing the token classification loss.
54
+ Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
55
+ are not taken into account for computing the loss.
56
+
57
+ Returns:
58
+
59
+ Example:
60
+
61
+ In this example below, we give the LayoutLMv2 model an image (of texts) and ask it a question. It will give us
62
+ a prediction of what it thinks the answer is (the span of the answer within the texts parsed from the image).
63
+
64
+ ```python
65
+ >>> from transformers import AutoTokenizer, LayoutLMForQuestionAnswering
66
+ >>> from datasets import load_dataset
67
+ >>> import torch
68
+
69
+ >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased", add_prefix_space=True)
70
+ >>> model = LayoutLMForQuestionAnswering.from_pretrained("microsoft/layoutlm-base-uncased")
71
+
72
+ >>> dataset = load_dataset("nielsr/funsd-layoutlmv3", split="train")
73
+ >>> example = dataset[0]
74
+ >>> question = "what's his name?"
75
+ >>> words = example["tokens"]
76
+ >>> boxes = example["bboxes"]
77
+
78
+ >>> encoding = tokenizer(
79
+ ... question.split(), words, is_split_into_words=True, return_token_type_ids=True, return_tensors="pt"
80
+ ... )
81
+ >>> bbox = []
82
+ >>> for i, s, w in zip(encoding.input_ids[0], encoding.sequence_ids(0), encoding.word_ids(0)):
83
+ ... if s == 1:
84
+ ... bbox.append(boxes[w])
85
+ ... elif i == tokenizer.sep_token_id:
86
+ ... bbox.append([1000] * 4)
87
+ ... else:
88
+ ... bbox.append([0] * 4)
89
+ >>> encoding["bbox"] = torch.tensor([bbox])
90
+
91
+ >>> outputs = model(**encoding)
92
+ >>> loss = outputs.loss
93
+ >>> start_scores = outputs.start_logits
94
+ >>> end_scores = outputs.end_logits
95
+ ```
96
+ """
97
+
98
+ return_dict = return_dict if return_dict is not None else self.config.use_return_dict
99
+
100
+ outputs = self.layoutlm(
101
+ input_ids=input_ids,
102
+ bbox=bbox,
103
+ attention_mask=attention_mask,
104
+ token_type_ids=token_type_ids,
105
+ position_ids=position_ids,
106
+ head_mask=head_mask,
107
+ inputs_embeds=inputs_embeds,
108
+ output_attentions=output_attentions,
109
+ output_hidden_states=output_hidden_states,
110
+ return_dict=return_dict,
111
+ )
112
+
113
+ sequence_output = outputs[0]
114
+
115
+ logits = self.qa_outputs(sequence_output)
116
+ start_logits, end_logits = logits.split(1, dim=-1)
117
+ start_logits = start_logits.squeeze(-1).contiguous()
118
+ end_logits = end_logits.squeeze(-1).contiguous()
119
+
120
+ total_loss = None
121
+ if start_positions is not None and end_positions is not None:
122
+ # If we are on multi-GPU, split add a dimension
123
+ if len(start_positions.size()) > 1:
124
+ start_positions = start_positions.squeeze(-1)
125
+ if len(end_positions.size()) > 1:
126
+ end_positions = end_positions.squeeze(-1)
127
+ # sometimes the start/end positions are outside our model inputs, we ignore these terms
128
+ ignored_index = start_logits.size(1)
129
+ start_positions = start_positions.clamp(0, ignored_index)
130
+ end_positions = end_positions.clamp(0, ignored_index)
131
+
132
+ loss_fct = CrossEntropyLoss(ignore_index=ignored_index)
133
+ start_loss = loss_fct(start_logits, start_positions)
134
+ end_loss = loss_fct(end_logits, end_positions)
135
+ total_loss = (start_loss + end_loss) / 2
136
+
137
+ if not return_dict:
138
+ output = (start_logits, end_logits) + outputs[2:]
139
+ return ((total_loss,) + output) if total_loss is not None else output
140
+
141
+ return QuestionAnsweringModelOutput(
142
+ loss=total_loss,
143
+ start_logits=start_logits,
144
+ end_logits=end_logits,
145
+ hidden_states=outputs.hidden_states,
146
+ attentions=outputs.attentions,
147
+ )
pipeline_document_question_answering.py ADDED
@@ -0,0 +1,375 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # NOTE: This code is currently under review for inclusion in the main
2
+ # huggingface/transformers repository:
3
+ # https://github.com/huggingface/transformers/pull/18414
4
+ from typing import List, Optional, Tuple, Union
5
+
6
+ import numpy as np
7
+
8
+ from transformers.utils import add_end_docstrings, is_torch_available, logging
9
+ from transformers.pipelines.base import PIPELINE_INIT_ARGS, Pipeline
10
+ from .qa_helpers import select_starts_ends, Image, load_image, VISION_LOADED, pytesseract, TESSERACT_LOADED
11
+
12
+
13
+ if is_torch_available():
14
+ import torch
15
+
16
+ # We do not perform the check in this version of the pipeline code
17
+ # from transformers.models.auto.modeling_auto import MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING
18
+
19
+ logger = logging.get_logger(__name__)
20
+
21
+
22
+ # normalize_bbox() and apply_tesseract() are derived from apply_tesseract in models/layoutlmv3/feature_extraction_layoutlmv3.py.
23
+ # However, because the pipeline may evolve from what layoutlmv3 currently does, it's copied (vs. imported) to avoid creating an
24
+ # unecessary dependency.
25
+ def normalize_box(box, width, height):
26
+ return [
27
+ int(1000 * (box[0] / width)),
28
+ int(1000 * (box[1] / height)),
29
+ int(1000 * (box[2] / width)),
30
+ int(1000 * (box[3] / height)),
31
+ ]
32
+
33
+
34
+ def apply_tesseract(image: "Image.Image", lang: Optional[str], tesseract_config: Optional[str]):
35
+ """Applies Tesseract OCR on a document image, and returns recognized words + normalized bounding boxes."""
36
+ # apply OCR
37
+ data = pytesseract.image_to_data(image, lang=lang, output_type="dict", config=tesseract_config)
38
+ words, left, top, width, height = data["text"], data["left"], data["top"], data["width"], data["height"]
39
+
40
+ # filter empty words and corresponding coordinates
41
+ irrelevant_indices = [idx for idx, word in enumerate(words) if not word.strip()]
42
+ words = [word for idx, word in enumerate(words) if idx not in irrelevant_indices]
43
+ left = [coord for idx, coord in enumerate(left) if idx not in irrelevant_indices]
44
+ top = [coord for idx, coord in enumerate(top) if idx not in irrelevant_indices]
45
+ width = [coord for idx, coord in enumerate(width) if idx not in irrelevant_indices]
46
+ height = [coord for idx, coord in enumerate(height) if idx not in irrelevant_indices]
47
+
48
+ # turn coordinates into (left, top, left+width, top+height) format
49
+ actual_boxes = []
50
+ for x, y, w, h in zip(left, top, width, height):
51
+ actual_box = [x, y, x + w, y + h]
52
+ actual_boxes.append(actual_box)
53
+
54
+ image_width, image_height = image.size
55
+
56
+ # finally, normalize the bounding boxes
57
+ normalized_boxes = []
58
+ for box in actual_boxes:
59
+ normalized_boxes.append(normalize_box(box, image_width, image_height))
60
+
61
+ assert len(words) == len(normalized_boxes), "Not as many words as there are bounding boxes"
62
+
63
+ return words, normalized_boxes
64
+
65
+
66
+ @add_end_docstrings(PIPELINE_INIT_ARGS)
67
+ class DocumentQuestionAnsweringPipeline(Pipeline):
68
+ # TODO: Update task_summary docs to include an example with document QA and then update the first sentence
69
+ """
70
+ Document Question Answering pipeline using any `AutoModelForDocumentQuestionAnswering`. See the [question answering
71
+ examples](../task_summary#question-answering) for more information.
72
+
73
+ This document question answering pipeline can currently be loaded from [`pipeline`] using the following task
74
+ identifier: `"document-question-answering"`.
75
+
76
+ The models that this pipeline can use are models that have been fine-tuned on a document question answering task.
77
+ See the up-to-date list of available models on
78
+ [huggingface.co/models](https://huggingface.co/models?filter=document-question-answering).
79
+ """
80
+
81
+ def __init__(self, *args, **kwargs):
82
+ super().__init__(*args, **kwargs)
83
+ # self.check_model_type(MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING)
84
+
85
+ def _sanitize_parameters(
86
+ self,
87
+ padding=None,
88
+ doc_stride=None,
89
+ max_question_len=None,
90
+ lang: Optional[str] = None,
91
+ tesseract_config: Optional[str] = None,
92
+ max_answer_len=None,
93
+ max_seq_len=None,
94
+ top_k=None,
95
+ handle_impossible_answer=None,
96
+ **kwargs,
97
+ ):
98
+ preprocess_params, postprocess_params = {}, {}
99
+ if padding is not None:
100
+ preprocess_params["padding"] = padding
101
+ if doc_stride is not None:
102
+ preprocess_params["doc_stride"] = doc_stride
103
+ if max_question_len is not None:
104
+ preprocess_params["max_question_len"] = max_question_len
105
+ if max_seq_len is not None:
106
+ preprocess_params["max_seq_len"] = max_seq_len
107
+ if lang is not None:
108
+ preprocess_params["lang"] = lang
109
+ if tesseract_config is not None:
110
+ preprocess_params["tesseract_config"] = tesseract_config
111
+
112
+ if top_k is not None:
113
+ if top_k < 1:
114
+ raise ValueError(f"top_k parameter should be >= 1 (got {top_k})")
115
+ postprocess_params["top_k"] = top_k
116
+ if max_answer_len is not None:
117
+ if max_answer_len < 1:
118
+ raise ValueError(f"max_answer_len parameter should be >= 1 (got {max_answer_len}")
119
+ postprocess_params["max_answer_len"] = max_answer_len
120
+ if handle_impossible_answer is not None:
121
+ postprocess_params["handle_impossible_answer"] = handle_impossible_answer
122
+
123
+ return preprocess_params, {}, postprocess_params
124
+
125
+ def __call__(
126
+ self,
127
+ image: Union["Image.Image", str],
128
+ question: Optional[str] = None,
129
+ word_boxes: Tuple[str, List[float]] = None,
130
+ **kwargs,
131
+ ):
132
+ """
133
+ Answer the question(s) given as inputs by using the document(s). A document is defined as an image and an
134
+ optional list of (word, box) tuples which represent the text in the document. If the `word_boxes` are not
135
+ provided, it will use the Tesseract OCR engine (if available) to extract the words and boxes automatically.
136
+
137
+ You can invoke the pipeline several ways:
138
+
139
+ - `pipeline(image=image, question=question)`
140
+ - `pipeline(image=image, question=question, word_boxes=word_boxes)`
141
+ - `pipeline([{"image": image, "question": question}])`
142
+ - `pipeline([{"image": image, "question": question, "word_boxes": word_boxes}])`
143
+
144
+ Args:
145
+ image (`str` or `PIL.Image`):
146
+ The pipeline handles three types of images:
147
+
148
+ - A string containing a http link pointing to an image
149
+ - A string containing a local path to an image
150
+ - An image loaded in PIL directly
151
+
152
+ The pipeline accepts either a single image or a batch of images. If given a single image, it can be
153
+ broadcasted to multiple questions.
154
+ question (`str`):
155
+ A question to ask of the document.
156
+ word_boxes (`List[str, Tuple[float, float, float, float]]`, *optional*):
157
+ A list of words and bounding boxes (normalized 0->1000). If you provide this optional input, then the
158
+ pipeline will use these words and boxes instead of running OCR on the image to derive them. This allows
159
+ you to reuse OCR'd results across many invocations of the pipeline without having to re-run it each
160
+ time.
161
+ top_k (`int`, *optional*, defaults to 1):
162
+ The number of answers to return (will be chosen by order of likelihood). Note that we return less than
163
+ top_k answers if there are not enough options available within the context.
164
+ doc_stride (`int`, *optional*, defaults to 128):
165
+ If the words in the document are too long to fit with the question for the model, it will be split in
166
+ several chunks with some overlap. This argument controls the size of that overlap.
167
+ max_answer_len (`int`, *optional*, defaults to 15):
168
+ The maximum length of predicted answers (e.g., only answers with a shorter length are considered).
169
+ max_seq_len (`int`, *optional*, defaults to 384):
170
+ The maximum length of the total sentence (context + question) in tokens of each chunk passed to the
171
+ model. The context will be split in several chunks (using `doc_stride` as overlap) if needed.
172
+ max_question_len (`int`, *optional*, defaults to 64):
173
+ The maximum length of the question after tokenization. It will be truncated if needed.
174
+ handle_impossible_answer (`bool`, *optional*, defaults to `False`):
175
+ Whether or not we accept impossible as an answer.
176
+ lang (`str`, *optional*):
177
+ Language to use while running OCR. Defaults to english.
178
+ tesseract_config (`str`, *optional*):
179
+ Additional flags to pass to tesseract while running OCR.
180
+
181
+ Return:
182
+ A `dict` or a list of `dict`: Each result comes as a dictionary with the following keys:
183
+
184
+ - **score** (`float`) -- The probability associated to the answer.
185
+ - **start** (`int`) -- The start word index of the answer (in the OCR'd version of the input or provided
186
+ `word_boxes`).
187
+ - **end** (`int`) -- The end word index of the answer (in the OCR'd version of the input or provided
188
+ `word_boxes`).
189
+ - **answer** (`str`) -- The answer to the question.
190
+ """
191
+ if isinstance(question, str):
192
+ inputs = {"question": question, "image": image, "word_boxes": word_boxes}
193
+ else:
194
+ inputs = image
195
+ return super().__call__(inputs, **kwargs)
196
+
197
+ def preprocess(
198
+ self,
199
+ input,
200
+ padding="do_not_pad",
201
+ doc_stride=None,
202
+ max_question_len=64,
203
+ max_seq_len=None,
204
+ word_boxes: Tuple[str, List[float]] = None,
205
+ lang=None,
206
+ tesseract_config="",
207
+ ):
208
+ # NOTE: This code mirrors the code in question answering and will be implemented in a follow up PR
209
+ # to support documents with enough tokens that overflow the model's window
210
+ # if max_seq_len is None:
211
+ # # TODO: LayoutLM's stride is 512 by default. Is it ok to use that as the min
212
+ # # instead of 384 (which the QA model uses)?
213
+ # max_seq_len = min(self.tokenizer.model_max_length, 512)
214
+
215
+ if doc_stride is not None:
216
+ # TODO implement
217
+ # doc_stride = min(max_seq_len // 2, 128)
218
+ raise ValueError("Unsupported: striding inputs")
219
+
220
+ image = None
221
+ image_features = {}
222
+ if "image" in input:
223
+ if not VISION_LOADED:
224
+ raise ValueError(
225
+ "If you provide an image, then the pipeline will run process it with PIL (Pillow), but"
226
+ " PIL is not available. Install it with pip install Pillow."
227
+ )
228
+ image = load_image(input["image"])
229
+ if self.feature_extractor is not None:
230
+ image_features.update(self.feature_extractor(images=image, return_tensors=self.framework))
231
+
232
+ words, boxes = None, None
233
+ if "word_boxes" in input:
234
+ words = [x[0] for x in input["word_boxes"]]
235
+ boxes = [x[1] for x in input["word_boxes"]]
236
+ elif "words" in image_features and "boxes" in image_features:
237
+ words = image_features.pop("words")
238
+ boxes = image_features.pop("boxes")
239
+ elif image is not None:
240
+ if not TESSERACT_LOADED:
241
+ raise ValueError(
242
+ "If you provide an image without word_boxes, then the pipeline will run OCR using Tesseract, but"
243
+ " pytesseract is not available. Install it with pip install pytesseract."
244
+ )
245
+ words, boxes = apply_tesseract(image, lang=lang, tesseract_config=tesseract_config)
246
+ else:
247
+ raise ValueError(
248
+ "You must provide an image or word_boxes. If you provide an image, the pipeline will automatically run"
249
+ " OCR to derive words and boxes"
250
+ )
251
+
252
+ if self.tokenizer.padding_side != "right":
253
+ raise ValueError(
254
+ "Document question answering only supports tokenizers whose padding side is 'right', not"
255
+ f" {self.tokenizer.padding_side}"
256
+ )
257
+
258
+ encoding = self.tokenizer(
259
+ text=input["question"].split(),
260
+ text_pair=words,
261
+ padding=padding,
262
+ max_length=max_seq_len,
263
+ stride=doc_stride,
264
+ return_token_type_ids=True,
265
+ is_split_into_words=True,
266
+ return_tensors=self.framework,
267
+ # TODO: In a future PR, use these feature to handle sequences whose length is longer than
268
+ # the maximum allowed by the model. Currently, the tokenizer will produce a sequence that
269
+ # may be too long for the model to handle.
270
+ # truncation="only_second",
271
+ # return_overflowing_tokens=True,
272
+ )
273
+ encoding.update(image_features)
274
+
275
+ # TODO: For now, this should always be num_spans == 1 given the flags we've passed in above, but the
276
+ # code is written to naturally handle multiple spans at the right time.
277
+ num_spans = len(encoding["input_ids"])
278
+
279
+ # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer)
280
+ # We put 0 on the tokens from the context and 1 everywhere else (question and special tokens)
281
+ # This logic mirrors the logic in the question_answering pipeline
282
+ p_mask = [[tok != 1 for tok in encoding.sequence_ids(span_id)] for span_id in range(num_spans)]
283
+ for span_idx in range(num_spans):
284
+ input_ids_span_idx = encoding["input_ids"][span_idx]
285
+ # keep the cls_token unmasked (some models use it to indicate unanswerable questions)
286
+ if self.tokenizer.cls_token_id is not None:
287
+ cls_indices = np.nonzero(np.array(input_ids_span_idx) == self.tokenizer.cls_token_id)[0]
288
+ for cls_index in cls_indices:
289
+ p_mask[span_idx][cls_index] = 0
290
+
291
+ # For each span, place a bounding box [0,0,0,0] for question and CLS tokens, [1000,1000,1000,1000]
292
+ # for SEP tokens, and the word's bounding box for words in the original document.
293
+ bbox = []
294
+ for batch_index in range(num_spans):
295
+ for i, s, w in zip(
296
+ encoding.input_ids[batch_index],
297
+ encoding.sequence_ids(batch_index),
298
+ encoding.word_ids(batch_index),
299
+ ):
300
+ if s == 1:
301
+ bbox.append(boxes[w])
302
+ elif i == self.tokenizer.sep_token_id:
303
+ bbox.append([1000] * 4)
304
+ else:
305
+ bbox.append([0] * 4)
306
+
307
+ if self.framework == "tf":
308
+ raise ValueError("Unsupported: Tensorflow preprocessing for DocumentQuestionAnsweringPipeline")
309
+ elif self.framework == "pt":
310
+ encoding["bbox"] = torch.tensor([bbox])
311
+
312
+ word_ids = [encoding.word_ids(i) for i in range(num_spans)]
313
+
314
+ # TODO This will be necessary when we implement overflow support
315
+ # encoding.pop("overflow_to_sample_mapping", None)
316
+
317
+ return {
318
+ **encoding,
319
+ "p_mask": p_mask,
320
+ "word_ids": word_ids,
321
+ "words": words,
322
+ }
323
+
324
+ def _forward(self, model_inputs):
325
+ p_mask = model_inputs.pop("p_mask", None)
326
+ word_ids = model_inputs.pop("word_ids", None)
327
+ words = model_inputs.pop("words", None)
328
+
329
+ model_outputs = self.model(**model_inputs)
330
+
331
+ model_outputs["p_mask"] = p_mask
332
+ model_outputs["word_ids"] = word_ids
333
+ model_outputs["words"] = words
334
+ model_outputs["attention_mask"] = model_inputs["attention_mask"]
335
+ return model_outputs
336
+
337
+ def postprocess(self, model_outputs, top_k=1, handle_impossible_answer=False, max_answer_len=15):
338
+ min_null_score = 1000000 # large and positive
339
+ answers = []
340
+ words = model_outputs["words"]
341
+
342
+ # TODO: Currently, we expect the length of model_outputs to be 1, because we do not stride
343
+ # in the preprocessor code. When we implement that, we'll either need to handle tensors of size
344
+ # > 1 or use the ChunkPipeline and handle multiple outputs (each of size = 1).
345
+ starts, ends, scores, min_null_score = select_starts_ends(
346
+ model_outputs["start_logits"],
347
+ model_outputs["end_logits"],
348
+ model_outputs["p_mask"],
349
+ model_outputs["attention_mask"].numpy() if model_outputs.get("attention_mask", None) is not None else None,
350
+ min_null_score,
351
+ top_k,
352
+ handle_impossible_answer,
353
+ max_answer_len,
354
+ )
355
+
356
+ word_ids = model_outputs["word_ids"][0]
357
+ for s, e, score in zip(starts, ends, scores):
358
+ word_start, word_end = word_ids[s], word_ids[e]
359
+ if word_start is not None and word_end is not None:
360
+ answers.append(
361
+ {
362
+ "score": score,
363
+ "answer": " ".join(words[word_start : word_end + 1]),
364
+ "start": word_start,
365
+ "end": word_end,
366
+ }
367
+ )
368
+
369
+ if handle_impossible_answer:
370
+ answers.append({"score": min_null_score, "answer": "", "start": 0, "end": 0})
371
+
372
+ answers = sorted(answers, key=lambda x: x["score"], reverse=True)[:top_k]
373
+ if len(answers) == 1:
374
+ return answers[0]
375
+ return answers
pytorch_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8870505d29315260ff57436aab0c66f3a2ddfb2cc7e09a2e368e04e762d0baba
3
+ size 511244837
qa_helpers.py ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # NOTE: This code is currently under review for inclusion in the main
2
+ # huggingface/transformers repository:
3
+ # https://github.com/huggingface/transformers/pull/18414
4
+
5
+ import warnings
6
+ from collections.abc import Iterable
7
+ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
8
+
9
+ import numpy as np
10
+
11
+ from transformers.utils import is_pytesseract_available, is_vision_available
12
+
13
+ VISION_LOADED = False
14
+ if is_vision_available():
15
+ from PIL import Image
16
+
17
+ from transformers.image_utils import load_image
18
+ VISION_LOADED = True
19
+ else:
20
+ Image = None
21
+ load_image = None
22
+
23
+
24
+ TESSERACT_LOADED = False
25
+ if is_pytesseract_available():
26
+ import pytesseract
27
+ TESSERACT_LOADED = True
28
+ else:
29
+ pytesseract = None
30
+
31
+ def decode_spans(
32
+ start: np.ndarray, end: np.ndarray, topk: int, max_answer_len: int, undesired_tokens: np.ndarray
33
+ ) -> Tuple:
34
+ """
35
+ Take the output of any `ModelForQuestionAnswering` and will generate probabilities for each span to be the actual
36
+ answer.
37
+
38
+ In addition, it filters out some unwanted/impossible cases like answer len being greater than max_answer_len or
39
+ answer end position being before the starting position. The method supports output the k-best answer through the
40
+ topk argument.
41
+
42
+ Args:
43
+ start (`np.ndarray`): Individual start probabilities for each token.
44
+ end (`np.ndarray`): Individual end probabilities for each token.
45
+ topk (`int`): Indicates how many possible answer span(s) to extract from the model output.
46
+ max_answer_len (`int`): Maximum size of the answer to extract from the model's output.
47
+ undesired_tokens (`np.ndarray`): Mask determining tokens that can be part of the answer
48
+ """
49
+ # Ensure we have batch axis
50
+ if start.ndim == 1:
51
+ start = start[None]
52
+
53
+ if end.ndim == 1:
54
+ end = end[None]
55
+
56
+ # Compute the score of each tuple(start, end) to be the real answer
57
+ outer = np.matmul(np.expand_dims(start, -1), np.expand_dims(end, 1))
58
+
59
+ # Remove candidate with end < start and end - start > max_answer_len
60
+ candidates = np.tril(np.triu(outer), max_answer_len - 1)
61
+
62
+ # Inspired by Chen & al. (https://github.com/facebookresearch/DrQA)
63
+ scores_flat = candidates.flatten()
64
+ if topk == 1:
65
+ idx_sort = [np.argmax(scores_flat)]
66
+ elif len(scores_flat) < topk:
67
+ idx_sort = np.argsort(-scores_flat)
68
+ else:
69
+ idx = np.argpartition(-scores_flat, topk)[0:topk]
70
+ idx_sort = idx[np.argsort(-scores_flat[idx])]
71
+
72
+ starts, ends = np.unravel_index(idx_sort, candidates.shape)[1:]
73
+ desired_spans = np.isin(starts, undesired_tokens.nonzero()) & np.isin(ends, undesired_tokens.nonzero())
74
+ starts = starts[desired_spans]
75
+ ends = ends[desired_spans]
76
+ scores = candidates[0, starts, ends]
77
+
78
+ return starts, ends, scores
79
+
80
+
81
+ def select_starts_ends(
82
+ start,
83
+ end,
84
+ p_mask,
85
+ attention_mask,
86
+ min_null_score=1000000,
87
+ top_k=1,
88
+ handle_impossible_answer=False,
89
+ max_answer_len=15,
90
+ ):
91
+ """
92
+ Takes the raw output of any `ModelForQuestionAnswering` and first normalizes its outputs and then uses
93
+ `decode_spans()` to generate probabilities for each span to be the actual answer.
94
+
95
+ Args:
96
+ start (`np.ndarray`): Individual start probabilities for each token.
97
+ end (`np.ndarray`): Individual end probabilities for each token.
98
+ p_mask (`np.ndarray`): A mask with 1 for values that cannot be in the answer
99
+ attention_mask (`np.ndarray`): The attention mask generated by the tokenizer
100
+ min_null_score(`float`): The minimum null (empty) answer score seen so far.
101
+ topk (`int`): Indicates how many possible answer span(s) to extract from the model output.
102
+ handle_impossible_answer(`bool`): Whether to allow null (empty) answers
103
+ max_answer_len (`int`): Maximum size of the answer to extract from the model's output.
104
+ """
105
+ # Ensure padded tokens & question tokens cannot belong to the set of candidate answers.
106
+ undesired_tokens = np.abs(np.array(p_mask) - 1)
107
+
108
+ if attention_mask is not None:
109
+ undesired_tokens = undesired_tokens & attention_mask
110
+
111
+ # Generate mask
112
+ undesired_tokens_mask = undesired_tokens == 0.0
113
+
114
+ # Make sure non-context indexes in the tensor cannot contribute to the softmax
115
+ start = np.where(undesired_tokens_mask, -10000.0, start)
116
+ end = np.where(undesired_tokens_mask, -10000.0, end)
117
+
118
+ # Normalize logits and spans to retrieve the answer
119
+ start = np.exp(start - start.max(axis=-1, keepdims=True))
120
+ start = start / start.sum()
121
+
122
+ end = np.exp(end - end.max(axis=-1, keepdims=True))
123
+ end = end / end.sum()
124
+
125
+ if handle_impossible_answer:
126
+ min_null_score = min(min_null_score, (start[0, 0] * end[0, 0]).item())
127
+
128
+ # Mask CLS
129
+ start[0, 0] = end[0, 0] = 0.0
130
+
131
+ starts, ends, scores = decode_spans(start, end, top_k, max_answer_len, undesired_tokens)
132
+ return starts, ends, scores, min_null_score
special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "sep_token": "</s>", "pad_token": "<pad>", "cls_token": "<s>", "mask_token": {"content": "<mask>", "single_word": false, "lstrip": true, "rstrip": false, "normalized": false}}
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_prefix_space": false, "errors": "replace", "sep_token": "</s>", "cls_token": "<s>", "pad_token": "<pad>", "mask_token": "<mask>", "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "roberta-base"}
vocab.json ADDED
The diff for this file is too large to render. See raw diff