initial commit

.gitattributes

@@ -0,0 +1,31 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,143 @@
+# dev files
+*.cache
+*.dev.py
+state_dict/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+*.pyc
+tests/
+*.result.json
+.idea/
+
+# Embedding
+glove.840B.300d.txt
+glove.42B.300d.txt
+glove.twitter.27B.txt
+
+# project main files
+release_note.json
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer training_logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+.DS_Store
+examples/.DS_Store

README.md

@@ -0,0 +1,13 @@
+---
+title: RPD-Demo
+emoji: 🛡️
+colorFrom: gray
+colorTo: green
+sdk: gradio
+sdk_version: 3.0.19
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

anonymous_demo/__init__.py

@@ -0,0 +1,5 @@
+__version__ = '1.0.0'
+
+__name__ = 'anonymous_demo'
+
+from anonymous_demo.functional import TADCheckpointManager

anonymous_demo/core/tad/classic/__bert__/README.MD

@@ -0,0 +1,3 @@
+## This is the simple migration from ABSA-PyTorch under MIT license
+
+Project Address: https://github.com/songyouwei/ABSA-PyTorch

anonymous_demo/core/tad/classic/__bert__/__init__.py

@@ -0,0 +1 @@
+from .models import *

anonymous_demo/core/tad/classic/__bert__/dataset_utils/data_utils_for_inference.py

@@ -0,0 +1,116 @@
+import tqdm
+from findfile import find_cwd_dir
+from torch.utils.data import Dataset
+from transformers import AutoTokenizer
+
+
+class Tokenizer4Pretraining:
+    def __init__(self, max_seq_len, opt, **kwargs):
+        if kwargs.pop('offline', False):
+            self.tokenizer = AutoTokenizer.from_pretrained(find_cwd_dir(opt.pretrained_bert.split('/')[-1]),
+                                                           do_lower_case='uncased' in opt.pretrained_bert)
+        else:
+            self.tokenizer = AutoTokenizer.from_pretrained(opt.pretrained_bert,
+                                                           do_lower_case='uncased' in opt.pretrained_bert)
+        self.max_seq_len = max_seq_len
+
+    def text_to_sequence(self, text, reverse=False, padding='post', truncating='post'):
+
+        return self.tokenizer.encode(text, truncation=True, padding='max_length', max_length=self.max_seq_len,
+                                     return_tensors='pt')
+
+
+class BERTTADDataset(Dataset):
+
+    def __init__(self, tokenizer, opt):
+        self.bert_baseline_input_colses = {
+            'bert': ['text_bert_indices']
+        }
+
+        self.tokenizer = tokenizer
+        self.opt = opt
+        self.all_data = []
+
+    def parse_sample(self, text):
+        return [text]
+
+    def prepare_infer_sample(self, text: str, ignore_error):
+        self.process_data(self.parse_sample(text), ignore_error=ignore_error)
+
+    def process_data(self, samples, ignore_error=True):
+        all_data = []
+        if len(samples) > 100:
+            it = tqdm.tqdm(samples, postfix='preparing text classification inference dataloader...')
+        else:
+            it = samples
+        for text in it:
+            try:
+                # handle for empty lines in inference datasets
+                if text is None or '' == text.strip():
+                    raise RuntimeError('Invalid Input!')
+
+                if '!ref!' in text:
+                    text, _, labels = text.strip().partition('!ref!')
+                    text = text.strip()
+                    if labels.count(',') == 2:
+                        label, is_adv, adv_train_label = labels.strip().split(',')
+                        label, is_adv, adv_train_label = label.strip(), is_adv.strip(), adv_train_label.strip()
+                    elif labels.count(',') == 1:
+                        label, is_adv = labels.strip().split(',')
+                        label, is_adv = label.strip(), is_adv.strip()
+                        adv_train_label = '-100'
+                    elif labels.count(',') == 0:
+                        label = labels.strip()
+                        adv_train_label = '-100'
+                        is_adv = '-100'
+                    else:
+                        label = '-100'
+                        adv_train_label = '-100'
+                        is_adv = '-100'
+
+                    label = int(label)
+                    adv_train_label = int(adv_train_label)
+                    is_adv = int(is_adv)
+
+                else:
+                    text = text.strip()
+                    label = -100
+                    adv_train_label = -100
+                    is_adv = -100
+
+                text_indices = self.tokenizer.text_to_sequence('{}'.format(text))
+
+                data = {
+                    'text_bert_indices': text_indices[0],
+
+                    'text_raw': text,
+
+                    'label': label,
+
+                    'adv_train_label': adv_train_label,
+
+                    'is_adv': is_adv,
+
+                    # 'label': self.opt.label_to_index.get(label, -100) if isinstance(label, str) else label,
+                    #
+                    # 'adv_train_label': self.opt.adv_train_label_to_index.get(adv_train_label, -100) if isinstance(adv_train_label, str) else adv_train_label,
+                    #
+                    # 'is_adv': self.opt.is_adv_to_index.get(is_adv, -100) if isinstance(is_adv, str) else is_adv,
+                }
+
+                all_data.append(data)
+
+            except Exception as e:
+                if ignore_error:
+                    print('Ignore error while processing:', text)
+                else:
+                    raise e
+
+        self.all_data = all_data
+        return self.all_data
+
+    def __getitem__(self, index):
+        return self.all_data[index]
+
+    def __len__(self):
+        return len(self.all_data)

anonymous_demo/core/tad/classic/__bert__/models/__init__.py

@@ -0,0 +1 @@
+from .tad_bert import TADBERT

anonymous_demo/core/tad/classic/__bert__/models/tad_bert.py

@@ -0,0 +1,43 @@
+import torch
+import torch.nn as nn
+from transformers.models.bert.modeling_bert import BertPooler
+
+from anonymous_demo.network.sa_encoder import Encoder
+
+
+class TADBERT(nn.Module):
+    inputs = ['text_bert_indices']
+
+    def __init__(self, bert, opt):
+        super(TADBERT, self).__init__()
+        self.opt = opt
+        self.bert = bert
+        self.pooler = BertPooler(bert.config)
+        self.dense1 = nn.Linear(self.opt.hidden_dim, self.opt.class_dim)
+        self.dense2 = nn.Linear(self.opt.hidden_dim, self.opt.adv_det_dim)
+        self.dense3 = nn.Linear(self.opt.hidden_dim, self.opt.class_dim)
+
+        self.encoder1 = Encoder(self.bert.config, opt=opt)
+        self.encoder2 = Encoder(self.bert.config, opt=opt)
+        self.encoder3 = Encoder(self.bert.config, opt=opt)
+
+    def forward(self, inputs):
+        text_raw_indices = inputs[0]
+        last_hidden_state = self.bert(text_raw_indices)['last_hidden_state']
+
+        sent_logits = self.dense1(self.pooler(last_hidden_state))
+        advdet_logits = self.dense2(self.pooler(last_hidden_state))
+        adv_tr_logits = self.dense3(self.pooler(last_hidden_state))
+
+        att_score = torch.nn.functional.normalize(
+            last_hidden_state.abs().sum(dim=1, keepdim=False) - last_hidden_state.abs().min(dim=1, keepdim=True)[0],
+            p=1, dim=1)
+
+        outputs = {
+            'sent_logits': sent_logits,
+            'advdet_logits': advdet_logits,
+            'adv_tr_logits': adv_tr_logits,
+            'last_hidden_state': last_hidden_state,
+            'att_score': att_score
+        }
+        return outputs

anonymous_demo/core/tad/models/__init__.py

@@ -0,0 +1,9 @@
+import anonymous_demo.core.tad.classic.__bert__.models
+
+
+class BERTTADModelList(list):
+    TADBERT = anonymous_demo.core.tad.classic.__bert__.TADBERT
+
+    def __init__(self):
+        model_list = [self.TADBERT]
+        super().__init__(model_list)

anonymous_demo/core/tad/prediction/tad_classifier.py

@@ -0,0 +1,390 @@
+import json
+import os
+import pickle
+import time
+
+import torch
+import tqdm
+from findfile import find_file, find_cwd_dir
+from termcolor import colored
+
+from torch.utils.data import DataLoader
+from transformers import AutoTokenizer, AutoModel, AutoConfig, DebertaV2ForMaskedLM, RobertaForMaskedLM, BertForMaskedLM
+
+from ....functional.dataset.dataset_manager import detect_infer_dataset
+
+from ..models import BERTTADModelList
+from ..classic.__bert__.dataset_utils.data_utils_for_inference import BERTTADDataset, Tokenizer4Pretraining
+
+from ....utils.demo_utils import print_args, TransformerConnectionError, get_device, build_embedding_matrix
+
+
+def init_attacker(tad_classifier, defense):
+    try:
+        from textattack import Attacker
+        from textattack.attack_recipes import BAEGarg2019, PWWSRen2019, TextFoolerJin2019, PSOZang2020, IGAWang2019, \
+            GeneticAlgorithmAlzantot2018, DeepWordBugGao2018
+        from textattack.datasets import Dataset
+        from textattack.models.wrappers import HuggingFaceModelWrapper
+
+        class DemoModelWrapper(HuggingFaceModelWrapper):
+            def __init__(self, model):
+                self.model = model  # pipeline = pipeline
+
+            def __call__(self, text_inputs, **kwargs):
+                outputs = []
+                for text_input in text_inputs:
+                    raw_outputs = self.model.infer(text_input, print_result=False, **kwargs)
+                    outputs.append(raw_outputs['probs'])
+                return outputs
+
+        class SentAttacker:
+
+            def __init__(self, model, recipe_class=BAEGarg2019):
+                model = model
+                model_wrapper = DemoModelWrapper(model)
+
+                recipe = recipe_class.build(model_wrapper)
+
+                _dataset = [('', 0)]
+                _dataset = Dataset(_dataset)
+
+                self.attacker = Attacker(recipe, _dataset)
+
+        attackers = {
+            'bae': BAEGarg2019,
+            'pwws': PWWSRen2019,
+            'textfooler': TextFoolerJin2019,
+            'pso': PSOZang2020,
+            'iga': IGAWang2019,
+            'ga': GeneticAlgorithmAlzantot2018,
+            'wordbugger': DeepWordBugGao2018,
+        }
+        return SentAttacker(tad_classifier, attackers[defense])
+    except Exception as e:
+
+        print('Original error:', e)
+
+
+def get_mlm_and_tokenizer(text_classifier, config):
+    if isinstance(text_classifier, TADTextClassifier):
+        base_model = text_classifier.model.bert.base_model
+    else:
+        base_model = text_classifier.bert.base_model
+    pretrained_config = AutoConfig.from_pretrained(config.pretrained_bert)
+    if 'deberta-v3' in config.pretrained_bert:
+        MLM = DebertaV2ForMaskedLM(pretrained_config)
+        MLM.deberta = base_model
+    elif 'roberta' in config.pretrained_bert:
+        MLM = RobertaForMaskedLM(pretrained_config)
+        MLM.roberta = base_model
+    else:
+        MLM = BertForMaskedLM(pretrained_config)
+        MLM.bert = base_model
+    return MLM, AutoTokenizer.from_pretrained(config.pretrained_bert)
+
+
+class TADTextClassifier:
+    def __init__(self, model_arg=None, cal_perplexity=False, **kwargs):
+        '''
+            from_train_model: load inference model from trained model
+        '''
+        self.cal_perplexity = cal_perplexity
+        # load from a training
+        if not isinstance(model_arg, str):
+            print('Load text classifier from training')
+            self.model = model_arg[0]
+            self.opt = model_arg[1]
+            self.tokenizer = model_arg[2]
+        else:
+            try:
+                if 'fine-tuned' in model_arg:
+                    raise ValueError(
+                        'Do not support to directly load a fine-tuned model, please load a .state_dict or .model instead!')
+                print('Load text classifier from', model_arg)
+                state_dict_path = find_file(model_arg, key='.state_dict', exclude_key=['__MACOSX'])
+                model_path = find_file(model_arg, key='.model', exclude_key=['__MACOSX'])
+                tokenizer_path = find_file(model_arg, key='.tokenizer', exclude_key=['__MACOSX'])
+                config_path = find_file(model_arg, key='.config', exclude_key=['__MACOSX'])
+
+                print('config: {}'.format(config_path))
+                print('state_dict: {}'.format(state_dict_path))
+                print('model: {}'.format(model_path))
+                print('tokenizer: {}'.format(tokenizer_path))
+
+                with open(config_path, mode='rb') as f:
+                    self.opt = pickle.load(f)
+                    self.opt.device = get_device(kwargs.pop('auto_device', True))[0]
+
+                if state_dict_path or model_path:
+                    if hasattr(BERTTADModelList, self.opt.model.__name__):
+                        if state_dict_path:
+                            if kwargs.pop('offline', False):
+                                self.bert = AutoModel.from_pretrained(
+                                    find_cwd_dir(self.opt.pretrained_bert.split('/')[-1]))
+                            else:
+                                self.bert = AutoModel.from_pretrained(self.opt.pretrained_bert)
+                            self.model = self.opt.model(self.bert, self.opt)
+                            self.model.load_state_dict(torch.load(state_dict_path, map_location='cpu'))
+                        elif model_path:
+                            self.model = torch.load(model_path, map_location='cpu')
+
+                        try:
+                            self.tokenizer = Tokenizer4Pretraining(max_seq_len=self.opt.max_seq_len, opt=self.opt,
+                                                                   **kwargs)
+                        except ValueError:
+                            if tokenizer_path:
+                                with open(tokenizer_path, mode='rb') as f:
+                                    self.tokenizer = pickle.load(f)
+                            else:
+                                raise TransformerConnectionError()
+
+            except Exception as e:
+                raise RuntimeError('Exception: {} Fail to load the model from {}! '.format(e, model_arg))
+
+        self.infer_dataloader = None
+        self.opt.eval_batch_size = kwargs.pop('eval_batch_size', 128)
+
+        self.opt.initializer = self.opt.initializer
+
+        if self.cal_perplexity:
+            try:
+                self.MLM, self.MLM_tokenizer = get_mlm_and_tokenizer(self, self.opt)
+            except Exception as e:
+                self.MLM, self.MLM_tokenizer = None, None
+
+        self.to(self.opt.device)
+
+    def to(self, device=None):
+        self.opt.device = device
+        self.model.to(device)
+        if hasattr(self, 'MLM'):
+            self.MLM.to(self.opt.device)
+
+    def cpu(self):
+        self.opt.device = 'cpu'
+        self.model.to('cpu')
+        if hasattr(self, 'MLM'):
+            self.MLM.to('cpu')
+
+    def cuda(self, device='cuda:0'):
+        self.opt.device = device
+        self.model.to(device)
+        if hasattr(self, 'MLM'):
+            self.MLM.to(device)
+
+    def _log_write_args(self):
+        n_trainable_params, n_nontrainable_params = 0, 0
+        for p in self.model.parameters():
+            n_params = torch.prod(torch.tensor(p.shape))
+            if p.requires_grad:
+                n_trainable_params += n_params
+            else:
+                n_nontrainable_params += n_params
+        print(
+            'n_trainable_params: {0}, n_nontrainable_params: {1}'.format(n_trainable_params, n_nontrainable_params))
+        for arg in vars(self.opt):
+            if getattr(self.opt, arg) is not None:
+                print('>>> {0}: {1}'.format(arg, getattr(self.opt, arg)))
+
+    def batch_infer(self,
+                    target_file=None,
+                    print_result=True,
+                    save_result=False,
+                    ignore_error=True,
+                    defense: str = None
+                    ):
+
+        save_path = os.path.join(os.getcwd(), 'tad_text_classification.result.json')
+
+        target_file = detect_infer_dataset(target_file, task='text_defense')
+        if not target_file:
+            raise FileNotFoundError('Can not find inference datasets!')
+
+        if hasattr(BERTTADModelList, self.opt.model.__name__):
+            dataset = BERTTADDataset(tokenizer=self.tokenizer, opt=self.opt)
+
+        dataset.prepare_infer_dataset(target_file, ignore_error=ignore_error)
+        self.infer_dataloader = DataLoader(dataset=dataset, batch_size=self.opt.eval_batch_size, pin_memory=True,
+                                           shuffle=False)
+        return self._infer(save_path=save_path if save_result else None, print_result=print_result, defense=defense)
+
+    def infer(self,
+              text: str = None,
+              print_result=True,
+              ignore_error=True,
+              defense: str = None
+              ):
+
+        if hasattr(BERTTADModelList, self.opt.model.__name__):
+            dataset = BERTTADDataset(tokenizer=self.tokenizer, opt=self.opt)
+
+        if text:
+            dataset.prepare_infer_sample(text, ignore_error=ignore_error)
+        else:
+            raise RuntimeError('Please specify your datasets path!')
+        self.infer_dataloader = DataLoader(dataset=dataset, batch_size=self.opt.eval_batch_size, shuffle=False)
+        return self._infer(print_result=print_result, defense=defense)[0]
+
+    def _infer(self, save_path=None, print_result=True, defense=None):
+
+        _params = filter(lambda p: p.requires_grad, self.model.parameters())
+
+        correct = {True: 'Correct', False: 'Wrong'}
+        results = []
+
+        with torch.no_grad():
+            self.model.eval()
+            n_correct = 0
+            n_labeled = 0
+
+            n_advdet_correct = 0
+            n_advdet_labeled = 0
+            if len(self.infer_dataloader.dataset) >= 100:
+                it = tqdm.tqdm(self.infer_dataloader, postfix='inferring...')
+            else:
+                it = self.infer_dataloader
+            for _, sample in enumerate(it):
+                inputs = [sample[col].to(self.opt.device) for col in self.opt.inputs_cols]
+                outputs = self.model(inputs)
+                logits, advdet_logits, adv_tr_logits = outputs['sent_logits'], outputs['advdet_logits'], outputs[
+                    'adv_tr_logits']
+                probs, advdet_probs, adv_tr_probs = torch.softmax(logits, dim=-1), torch.softmax(advdet_logits,
+                                                                                                 dim=-1), torch.softmax(
+                    adv_tr_logits, dim=-1)
+
+                for i, (prob, advdet_prob, adv_tr_prob) in enumerate(zip(probs, advdet_probs, adv_tr_probs)):
+                    text_raw = sample['text_raw'][i]
+
+                    pred_label = int(prob.argmax(axis=-1))
+                    pred_is_adv_label = int(advdet_prob.argmax(axis=-1))
+                    pred_adv_tr_label = int(adv_tr_prob.argmax(axis=-1))
+                    ref_label = int(sample['label'][i]) if int(sample['label'][i]) in self.opt.index_to_label else ''
+                    ref_is_adv_label = int(sample['is_adv'][i]) if int(
+                        sample['is_adv'][i]) in self.opt.index_to_is_adv else ''
+                    ref_adv_tr_label = int(sample['adv_train_label'][i]) if int(
+                        sample['adv_train_label'][i]) in self.opt.index_to_adv_train_label else ''
+
+                    if self.cal_perplexity:
+                        ids = self.MLM_tokenizer(text_raw, return_tensors="pt")
+                        ids['labels'] = ids['input_ids'].clone()
+                        ids = ids.to(self.opt.device)
+                        loss = self.MLM(**ids)['loss']
+                        perplexity = float(torch.exp(loss / ids['input_ids'].size(1)))
+                    else:
+                        perplexity = 'N.A.'
+
+                    result = {
+                        'text': text_raw,
+
+                        'label': self.opt.index_to_label[pred_label],
+                        'probs': prob.cpu().numpy(),
+                        'confidence': float(max(prob)),
+                        'ref_label': self.opt.index_to_label[ref_label] if isinstance(ref_label, int) else ref_label,
+                        'ref_label_check': correct[pred_label == ref_label] if ref_label != -100 else '',
+                        'is_fixed': False,
+
+                        'is_adv_label': self.opt.index_to_is_adv[pred_is_adv_label],
+                        'is_adv_probs': advdet_prob.cpu().numpy(),
+                        'is_adv_confidence': float(max(advdet_prob)),
+                        'ref_is_adv_label': self.opt.index_to_is_adv[ref_is_adv_label] if isinstance(ref_is_adv_label, int) else ref_is_adv_label,
+                        'ref_is_adv_check': correct[pred_is_adv_label == ref_is_adv_label] if ref_is_adv_label != -100 and isinstance(ref_is_adv_label, int) else '',
+
+                        'pred_adv_tr_label': self.opt.index_to_label[pred_adv_tr_label],
+                        'ref_adv_tr_label': self.opt.index_to_label[ref_adv_tr_label],
+
+                        'perplexity': perplexity,
+                    }
+                    if defense:
+                        try:
+                            if not hasattr(self, 'sent_attacker'):
+                                self.sent_attacker = init_attacker(self, defense.lower())
+                            if result['is_adv_label'] == '1':
+                                res = self.sent_attacker.attacker.simple_attack(text_raw, int(result['label']))
+                                new_infer_res = self.infer(res.perturbed_result.attacked_text.text, print_result=False)
+                                result['perturbed_label'] = result['label']
+                                result['label'] = new_infer_res['label']
+                                result['probs'] = new_infer_res['probs']
+                                result['ref_label_check'] = correct[int(result['label']) == ref_label] if ref_label != -100 else ''
+                                result['restored_text'] = res.perturbed_result.attacked_text.text
+                                result['is_fixed'] = True
+                            else:
+                                result['restored_text'] = ''
+                                result['is_fixed'] = False
+
+                        except Exception as e:
+                            print('Error:{}, try install TextAttack and tensorflow_text after 10 seconds...'.format(e))
+                            time.sleep(10)
+                            raise RuntimeError('Installation done, please run again...')
+
+                    if ref_label != -100:
+                        n_labeled += 1
+
+                        if result['label'] == result['ref_label']:
+                            n_correct += 1
+
+                    if ref_is_adv_label != -100:
+                        n_advdet_labeled += 1
+                        if ref_is_adv_label == pred_is_adv_label:
+                            n_advdet_correct += 1
+
+                    results.append(result)
+
+        try:
+            if print_result:
+                for ex_id, result in enumerate(results):
+                    text_printing = result['text'][:]
+                    text_info = ''
+                    if result['label'] != '-100':
+                        if not result['ref_label']:
+                            text_info += ' -> <CLS:{}(ref:{} confidence:{})>'.format(result['label'],
+                                                                                     result['ref_label'],
+                                                                                     result['confidence'])
+                        elif result['label'] == result['ref_label']:
+                            text_info += colored(
+                                ' -> <CLS:{}(ref:{} confidence:{})>'.format(result['label'], result['ref_label'],
+                                                                            result['confidence']), 'green')
+                        else:
+                            text_info += colored(
+                                ' -> <CLS:{}(ref:{} confidence:{})>'.format(result['label'], result['ref_label'],
+                                                                            result['confidence']), 'red')
+
+                    # AdvDet
+                    if result['is_adv_label'] != '-100':
+                        if not result['ref_is_adv_label']:
+                            text_info += ' -> <AdvDet:{}(ref:{} confidence:{})>'.format(result['is_adv_label'],
+                                                                                        result['ref_is_adv_check'],
+                                                                                        result['is_adv_confidence'])
+                        elif result['is_adv_label'] == result['ref_is_adv_label']:
+                            text_info += colored(' -> <AdvDet:{}(ref:{} confidence:{})>'.format(result['is_adv_label'],
+                                                                                                result[
+                                                                                                    'ref_is_adv_label'],
+                                                                                                result[
+                                                                                                    'is_adv_confidence']),
+                                                 'green')
+                        else:
+                            text_info += colored(' -> <AdvDet:{}(ref:{} confidence:{})>'.format(result['is_adv_label'],
+                                                                                                result[
+                                                                                                    'ref_is_adv_label'],
+                                                                                                result[
+                                                                                                    'is_adv_confidence']),
+                                                 'red')
+                    text_printing += text_info
+                    if self.cal_perplexity:
+                        text_printing += colored(' --> <perplexity:{}>'.format(result['perplexity']), 'yellow')
+                    print('Example {}: {}'.format(ex_id, text_printing))
+            if save_path:
+                with open(save_path, 'w', encoding='utf8') as fout:
+                    json.dump(str(results), fout, ensure_ascii=False)
+                    print('inference result saved in: {}'.format(save_path))
+        except Exception as e:
+            print('Can not save result: {}, Exception: {}'.format(text_raw, e))
+
+        if len(results) > 1:
+            print('CLS Acc:{}%'.format(100 * n_correct / n_labeled if n_labeled else ''))
+            print('AdvDet Acc:{}%'.format(100 * n_advdet_correct / n_advdet_labeled if n_advdet_labeled else ''))
+
+        return results
+
+    def clear_input_samples(self):
+        self.dataset.all_data = []

anonymous_demo/functional/__init__.py

@@ -0,0 +1,3 @@
+from anonymous_demo.functional.checkpoint.checkpoint_manager import TADCheckpointManager
+
+from anonymous_demo.functional.config import TADConfigManager

anonymous_demo/functional/checkpoint/__init__.py

@@ -0,0 +1 @@
+from .checkpoint_manager import TADCheckpointManager

anonymous_demo/functional/checkpoint/checkpoint_manager.py

@@ -0,0 +1,20 @@
+import os
+from findfile import find_file
+
+from anonymous_demo.core.tad.prediction.tad_classifier import TADTextClassifier
+from anonymous_demo.utils.demo_utils import retry
+
+
+class CheckpointManager:
+    pass
+
+
+class TADCheckpointManager(CheckpointManager):
+    @staticmethod
+    @retry
+    def get_tad_text_classifier(checkpoint: str = None,
+                                eval_batch_size=128,
+                                **kwargs):
+
+        tad_text_classifier = TADTextClassifier(checkpoint, eval_batch_size=eval_batch_size, **kwargs)
+        return tad_text_classifier

anonymous_demo/functional/config/__init__.py

@@ -0,0 +1 @@
+from .tad_config_manager import TADConfigManager

anonymous_demo/functional/config/config_manager.py

@@ -0,0 +1,66 @@
+from argparse import Namespace
+
+import torch
+
+one_shot_messages = set()
+
+
+def config_check(args):
+    pass
+
+
+class ConfigManager(Namespace):
+
+    def __init__(self, args=None, **kwargs):
+        """
+        The ConfigManager is a subclass of argparse.Namespace and based on parameter dict and count the call-frequency of each parameter
+        :param args: A parameter dict
+        :param kwargs: Same param as Namespce
+        """
+        if not args:
+            args = {}
+        super().__init__(**kwargs)
+
+        if isinstance(args, Namespace):
+            self.args = vars(args)
+            self.args_call_count = {arg: 0 for arg in vars(args)}
+        else:
+            self.args = args
+            self.args_call_count = {arg: 0 for arg in args}
+
+    def __getattribute__(self, arg_name):
+        if arg_name == 'args' or arg_name == 'args_call_count':
+            return super().__getattribute__(arg_name)
+        try:
+            value = super().__getattribute__('args')[arg_name]
+            args_call_count = super().__getattribute__('args_call_count')
+            args_call_count[arg_name] += 1
+            super().__setattr__('args_call_count', args_call_count)
+            return value
+
+        except Exception as e:
+
+            return super().__getattribute__(arg_name)
+
+    def __setattr__(self, arg_name, value):
+        if arg_name == 'args' or arg_name == 'args_call_count':
+            super().__setattr__(arg_name, value)
+            return
+        try:
+            args = super().__getattribute__('args')
+            args[arg_name] = value
+            super().__setattr__('args', args)
+            args_call_count = super().__getattribute__('args_call_count')
+
+            if arg_name in args_call_count:
+                # args_call_count[arg_name] += 1
+                super().__setattr__('args_call_count', args_call_count)
+
+            else:
+                args_call_count[arg_name] = 0
+                super().__setattr__('args_call_count', args_call_count)
+
+        except Exception as e:
+            super().__setattr__(arg_name, value)
+
+        config_check(args)

anonymous_demo/functional/config/tad_config_manager.py

@@ -0,0 +1,221 @@
+import copy
+
+from anonymous_demo.functional.config.config_manager import ConfigManager
+from anonymous_demo.core.tad.classic.__bert__.models import TADBERT
+
+_tad_config_template = {'model': TADBERT,
+                        'optimizer': "adamw",
+                        'learning_rate': 0.00002,
+                        'patience': 99999,
+                        'pretrained_bert': "microsoft/mdeberta-v3-base",
+                        'cache_dataset': True,
+                        'warmup_step': -1,
+                        'show_metric': False,
+                        'max_seq_len': 80,
+                        'dropout': 0,
+                        'l2reg': 0.000001,
+                        'num_epoch': 10,
+                        'batch_size': 16,
+                        'initializer': 'xavier_uniform_',
+                        'seed': 52,
+                        'polarities_dim': 3,
+                        'log_step': 10,
+                        'evaluate_begin': 0,
+                        'cross_validate_fold': -1,
+                        'use_amp': False,
+                        # split train and test datasets into 5 folds and repeat 3 training
+                        }
+
+_tad_config_base = {'model': TADBERT,
+                    'optimizer': "adamw",
+                    'learning_rate': 0.00002,
+                    'pretrained_bert': "microsoft/deberta-v3-base",
+                    'cache_dataset': True,
+                    'warmup_step': -1,
+                    'show_metric': False,
+                    'max_seq_len': 80,
+                    'patience': 99999,
+                    'dropout': 0,
+                    'l2reg': 0.000001,
+                    'num_epoch': 10,
+                    'batch_size': 16,
+                    'initializer': 'xavier_uniform_',
+                    'seed': 52,
+                    'polarities_dim': 3,
+                    'log_step': 10,
+                    'evaluate_begin': 0,
+                    'cross_validate_fold': -1
+                    # split train and test datasets into 5 folds and repeat 3 training
+                    }
+
+_tad_config_english = {'model': TADBERT,
+                       'optimizer': "adamw",
+                       'learning_rate': 0.00002,
+                       'patience': 99999,
+                       'pretrained_bert': "microsoft/deberta-v3-base",
+                       'cache_dataset': True,
+                       'warmup_step': -1,
+                       'show_metric': False,
+                       'max_seq_len': 80,
+                       'dropout': 0,
+                       'l2reg': 0.000001,
+                       'num_epoch': 10,
+                       'batch_size': 16,
+                       'initializer': 'xavier_uniform_',
+                       'seed': 52,
+                       'polarities_dim': 3,
+                       'log_step': 10,
+                       'evaluate_begin': 0,
+                       'cross_validate_fold': -1
+                       # split train and test datasets into 5 folds and repeat 3 training
+                       }
+
+_tad_config_multilingual = {'model': TADBERT,
+                            'optimizer': "adamw",
+                            'learning_rate': 0.00002,
+                            'patience': 99999,
+                            'pretrained_bert': "microsoft/mdeberta-v3-base",
+                            'cache_dataset': True,
+                            'warmup_step': -1,
+                            'show_metric': False,
+                            'max_seq_len': 80,
+                            'dropout': 0,
+                            'l2reg': 0.000001,
+                            'num_epoch': 10,
+                            'batch_size': 16,
+                            'initializer': 'xavier_uniform_',
+                            'seed': 52,
+                            'polarities_dim': 3,
+                            'log_step': 10,
+                            'evaluate_begin': 0,
+                            'cross_validate_fold': -1
+                            # split train and test datasets into 5 folds and repeat 3 training
+                            }
+
+_tad_config_chinese = {'model': TADBERT,
+                       'optimizer': "adamw",
+                       'learning_rate': 0.00002,
+                       'patience': 99999,
+                       'cache_dataset': True,
+                       'warmup_step': -1,
+                       'show_metric': False,
+                       'pretrained_bert': "bert-base-chinese",
+                       'max_seq_len': 80,
+                       'dropout': 0,
+                       'l2reg': 0.000001,
+                       'num_epoch': 10,
+                       'batch_size': 16,
+                       'initializer': 'xavier_uniform_',
+                       'seed': 52,
+                       'polarities_dim': 3,
+                       'log_step': 10,
+                       'evaluate_begin': 0,
+                       'cross_validate_fold': -1
+                       # split train and test datasets into 5 folds and repeat 3 training
+                       }
+
+
+class TADConfigManager(ConfigManager):
+    def __init__(self, args, **kwargs):
+        """
+        Available Params:  {'model': BERT,
+                            'optimizer': "adamw",
+                            'learning_rate': 0.00002,
+                            'pretrained_bert': "roberta-base",
+                            'cache_dataset': True,
+                            'warmup_step': -1,
+                            'show_metric': False,
+                            'max_seq_len': 80,
+                            'patience': 99999,
+                            'dropout': 0,
+                            'l2reg': 0.000001,
+                            'num_epoch': 10,
+                            'batch_size': 16,
+                            'initializer': 'xavier_uniform_',
+                            'seed': {52, 25}
+                            'embed_dim': 768,
+                            'hidden_dim': 768,
+                            'polarities_dim': 3,
+                            'log_step': 10,
+                            'evaluate_begin': 0,
+                            'cross_validate_fold': -1 # split train and test datasets into 5 folds and repeat 3 training
+                            }
+        :param args:
+        :param kwargs:
+        """
+        super().__init__(args, **kwargs)
+
+    @staticmethod
+    def set_tad_config(configType: str, newitem: dict):
+        if isinstance(newitem, dict):
+            if configType == 'template':
+                _tad_config_template.update(newitem)
+            elif configType == 'base':
+                _tad_config_base.update(newitem)
+            elif configType == 'english':
+                _tad_config_english.update(newitem)
+            elif configType == 'chinese':
+                _tad_config_chinese.update(newitem)
+            elif configType == 'multilingual':
+                _tad_config_multilingual.update(newitem)
+            elif configType == 'glove':
+                _tad_config_glove.update(newitem)
+            else:
+                raise ValueError(
+                    "Wrong value of config type supplied, please use one from following type: template, base, english, chinese, multilingual, glove")
+        else:
+            raise TypeError("Wrong type of new config item supplied, please use dict e.g.{'NewConfig': NewValue}")
+
+    @staticmethod
+    def set_tad_config_template(newitem):
+        TADConfigManager.set_tad_config('template', newitem)
+
+    @staticmethod
+    def set_tad_config_base(newitem):
+        TADConfigManager.set_tad_config('base', newitem)
+
+    @staticmethod
+    def set_tad_config_english(newitem):
+        TADConfigManager.set_tad_config('english', newitem)
+
+    @staticmethod
+    def set_tad_config_chinese(newitem):
+        TADConfigManager.set_tad_config('chinese', newitem)
+
+    @staticmethod
+    def set_tad_config_multilingual(newitem):
+        TADConfigManager.set_tad_config('multilingual', newitem)
+
+    @staticmethod
+    def set_tad_config_glove(newitem):
+        TADConfigManager.set_tad_config('glove', newitem)
+
+    @staticmethod
+    def get_tad_config_template() -> ConfigManager:
+        _tad_config_template.update(_tad_config_template)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_base() -> ConfigManager:
+        _tad_config_template.update(_tad_config_base)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_english() -> ConfigManager:
+        _tad_config_template.update(_tad_config_english)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_chinese() -> ConfigManager:
+        _tad_config_template.update(_tad_config_chinese)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_multilingual() -> ConfigManager:
+        _tad_config_template.update(_tad_config_multilingual)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))
+
+    @staticmethod
+    def get_tad_config_glove() -> ConfigManager:
+        _tad_config_template.update(_tad_config_glove)
+        return TADConfigManager(copy.deepcopy(_tad_config_template))

anonymous_demo/functional/dataset/__init__.py

@@ -0,0 +1 @@
+from anonymous_demo.functional.dataset.dataset_manager import (detect_infer_dataset)

anonymous_demo/functional/dataset/dataset_manager.py

@@ -0,0 +1,21 @@
+import os
+from findfile import find_files, find_dir
+
+filter_key_words = ['.py', '.md', 'readme', 'log', 'result', 'zip',
+                    '.state_dict', '.model', '.png', 'acc_', 'f1_', '.backup', '.bak']
+
+
+def detect_infer_dataset(dataset_path, task='apc'):
+    dataset_file = []
+    if isinstance(dataset_path, str) and os.path.isfile(dataset_path):
+        dataset_file.append(dataset_path)
+        return dataset_file
+
+    for d in dataset_path:
+        if not os.path.exists(d):
+            search_path = find_dir(os.getcwd(), [d, task, 'dataset'], exclude_key=filter_key_words, disable_alert=False)
+            dataset_file += find_files(search_path, ['.inference', d], exclude_key=['train.'] + filter_key_words)
+        else:
+            dataset_file += find_files(d, ['.inference', task], exclude_key=['train.'] + filter_key_words)
+
+    return dataset_file

anonymous_demo/network/lcf_pooler.py

@@ -0,0 +1,26 @@
+import numpy
+import torch
+import torch.nn as nn
+
+
+class LCF_Pooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = nn.Tanh()
+
+    def forward(self, hidden_states, lcf_vec):
+        device = hidden_states.device
+        lcf_vec = lcf_vec.detach().cpu().numpy()
+
+        pooled_output = numpy.zeros((hidden_states.shape[0], hidden_states.shape[2]), dtype=numpy.float32)
+        hidden_states = hidden_states.detach().cpu().numpy()
+        for i, vec in enumerate(lcf_vec):
+            lcf_ids = [j for j in range(len(vec)) if sum(vec[j] - 1.) == 0]
+            pooled_output[i] = hidden_states[i][lcf_ids[len(lcf_ids) // 2]]
+
+        pooled_output = torch.Tensor(pooled_output).to(device)
+        pooled_output = self.dense(pooled_output)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output

anonymous_demo/network/lsa.py

@@ -0,0 +1,52 @@
+import torch
+from anonymous_demo.network.sa_encoder import Encoder
+from torch import nn
+
+
+class LSA(nn.Module):
+    def __init__(self, bert, opt):
+        super(LSA, self).__init__()
+        self.opt = opt
+
+        self.encoder = Encoder(bert.config, opt)
+        self.encoder_left = Encoder(bert.config, opt)
+        self.encoder_right = Encoder(bert.config, opt)
+        self.linear_window_3h = nn.Linear(opt.embed_dim * 3, opt.embed_dim)
+        self.linear_window_2h = nn.Linear(opt.embed_dim * 2, opt.embed_dim)
+        self.eta1 = nn.Parameter(torch.tensor(self.opt.eta, dtype=torch.float))
+        self.eta2 = nn.Parameter(torch.tensor(self.opt.eta, dtype=torch.float))
+
+    def forward(self, global_context_features, spc_mask_vec, lcf_matrix, left_lcf_matrix, right_lcf_matrix):
+        masked_global_context_features = torch.mul(spc_mask_vec, global_context_features)
+
+        # # --------------------------------------------------- #
+        lcf_features = torch.mul(global_context_features, lcf_matrix)
+        lcf_features = self.encoder(lcf_features)
+        # # --------------------------------------------------- #
+        left_lcf_features = torch.mul(masked_global_context_features, left_lcf_matrix)
+        left_lcf_features = self.encoder_left(left_lcf_features)
+        # # --------------------------------------------------- #
+        right_lcf_features = torch.mul(masked_global_context_features, right_lcf_matrix)
+        right_lcf_features = self.encoder_right(right_lcf_features)
+        # # --------------------------------------------------- #
+        if 'lr' == self.opt.window or 'rl' == self.opt.window:
+            if self.eta1 <= 0 and self.opt.eta != -1:
+                torch.nn.init.uniform_(self.eta1)
+                print('reset eta1 to: {}'.format(self.eta1.item()))
+            if self.eta2 <= 0 and self.opt.eta != -1:
+                torch.nn.init.uniform_(self.eta2)
+                print('reset eta2 to: {}'.format(self.eta2.item()))
+            if self.opt.eta >= 0:
+                cat_features = torch.cat((lcf_features, self.eta1 * left_lcf_features, self.eta2 * right_lcf_features),
+                                         -1)
+            else:
+                cat_features = torch.cat((lcf_features, left_lcf_features, right_lcf_features), -1)
+            sent_out = self.linear_window_3h(cat_features)
+        elif 'l' == self.opt.window:
+            sent_out = self.linear_window_2h(torch.cat((lcf_features, self.eta1 * left_lcf_features), -1))
+        elif 'r' == self.opt.window:
+            sent_out = self.linear_window_2h(torch.cat((lcf_features, self.eta2 * right_lcf_features), -1))
+        else:
+            raise KeyError('Invalid parameter:', self.opt.window)
+
+        return sent_out

anonymous_demo/network/sa_encoder.py

@@ -0,0 +1,159 @@
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+
+class BertSelfAttention(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"):
+            raise ValueError(
+                f"The hidden size ({config.hidden_size}) is not a multiple of the number of attention "
+                f"heads ({config.num_attention_heads})"
+            )
+
+        self.num_attention_heads = config.num_attention_heads
+        self.attention_head_size = int(config.hidden_size / config.num_attention_heads)
+        self.all_head_size = self.num_attention_heads * self.attention_head_size
+
+        self.query = nn.Linear(config.hidden_size, self.all_head_size)
+        self.key = nn.Linear(config.hidden_size, self.all_head_size)
+        self.value = nn.Linear(config.hidden_size, self.all_head_size)
+
+        self.dropout = nn.Dropout(
+            config.attention_probs_dropout_prob if hasattr(config, 'attention_probs_dropout_prob') else 0)
+        self.position_embedding_type = getattr(config, "position_embedding_type", "absolute")
+        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
+            self.max_position_embeddings = config.max_position_embeddings
+            self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size)
+
+        self.is_decoder = config.is_decoder
+
+    def transpose_for_scores(self, x):
+        new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size)
+        x = x.view(*new_x_shape)
+        return x.permute(0, 2, 1, 3)
+
+    def forward(
+        self,
+        hidden_states,
+        attention_mask=None,
+        head_mask=None,
+        encoder_hidden_states=None,
+        encoder_attention_mask=None,
+        past_key_value=None,
+        output_attentions=False,
+    ):
+        mixed_query_layer = self.query(hidden_states)
+
+        # If this is instantiated as a cross-attention module, the keys
+        # and values come from an encoder; the attention mask needs to be
+        # such that the encoder's padding tokens are not attended to.
+        is_cross_attention = encoder_hidden_states is not None
+
+        if is_cross_attention and past_key_value is not None:
+            # reuse k,v, cross_attentions
+            key_layer = past_key_value[0]
+            value_layer = past_key_value[1]
+            attention_mask = encoder_attention_mask
+        elif is_cross_attention:
+            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
+            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
+            attention_mask = encoder_attention_mask
+        elif past_key_value is not None:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+            key_layer = torch.cat([past_key_value[0], key_layer], dim=2)
+            value_layer = torch.cat([past_key_value[1], value_layer], dim=2)
+        else:
+            key_layer = self.transpose_for_scores(self.key(hidden_states))
+            value_layer = self.transpose_for_scores(self.value(hidden_states))
+
+        query_layer = self.transpose_for_scores(mixed_query_layer)
+
+        if self.is_decoder:
+            # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states.
+            # Further calls to cross_attention layer can then reuse all cross-attention
+            # key/value_states (first "if" case)
+            # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of
+            # all previous decoder key/value_states. Further calls to uni-directional self-attention
+            # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case)
+            # if encoder bi-directional self-attention `past_key_value` is always `None`
+            past_key_value = (key_layer, value_layer)
+
+        # Take the dot product between "query" and "key" to get the raw attention scores.
+        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
+
+        if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query":
+            seq_length = hidden_states.size()[1]
+            position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1)
+            position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1)
+            distance = position_ids_l - position_ids_r
+            positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1)
+            positional_embedding = positional_embedding.to(dtype=query_layer.dtype)  # fp16 compatibility
+
+            if self.position_embedding_type == "relative_key":
+                relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
+                attention_scores = attention_scores + relative_position_scores
+            elif self.position_embedding_type == "relative_key_query":
+                relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding)
+                relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding)
+                attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key
+
+        attention_scores = attention_scores / math.sqrt(self.attention_head_size)
+        if attention_mask is not None:
+            # Apply the attention mask is (precomputed for all layers in BertModel forward() function)
+            attention_scores = attention_scores + attention_mask
+
+        # Normalize the attention scores to probabilities.
+        attention_probs = nn.Softmax(dim=-1)(attention_scores)
+
+        # This is actually dropping out entire tokens to attend to, which might
+        # seem a bit unusual, but is taken from the original Transformer paper.
+        attention_probs = self.dropout(attention_probs)
+
+        # Mask heads if we want to
+        if head_mask is not None:
+            attention_probs = attention_probs * head_mask
+
+        context_layer = torch.matmul(attention_probs, value_layer)
+
+        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
+        new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,)
+        context_layer = context_layer.view(*new_context_layer_shape)
+
+        outputs = (context_layer, attention_probs) if output_attentions else (context_layer,)
+
+        if self.is_decoder:
+            outputs = outputs + (past_key_value,)
+        return outputs
+
+
+class Encoder(nn.Module):
+    def __init__(self, config, opt, layer_num=1):
+        super(Encoder, self).__init__()
+        self.opt = opt
+        self.config = config
+        self.encoder = nn.ModuleList([SelfAttention(config, opt) for _ in range(layer_num)])
+        self.tanh = torch.nn.Tanh()
+
+    def forward(self, x):
+        for i, enc in enumerate(self.encoder):
+            x = self.tanh(enc(x)[0])
+        return x
+
+
+class SelfAttention(nn.Module):
+    def __init__(self, config, opt):
+        super(SelfAttention, self).__init__()
+        self.opt = opt
+        self.config = config
+        self.SA = BertSelfAttention(config)
+
+    def forward(self, inputs):
+        zero_vec = np.zeros((inputs.size(0), 1, 1, self.opt.max_seq_len))
+        zero_tensor = torch.tensor(zero_vec).float().to(inputs.device)
+        SA_out = self.SA(inputs, zero_tensor)
+        return SA_out

anonymous_demo/utils/demo_utils.py

@@ -0,0 +1,209 @@
+import json
+import os
+import pickle
+import signal
+import threading
+import time
+import zipfile
+
+import gdown
+import numpy as np
+import requests
+import torch
+import tqdm
+from autocuda import auto_cuda, auto_cuda_name
+from findfile import find_files, find_cwd_file, find_file
+from termcolor import colored
+from functools import wraps
+
+from update_checker import parse_version
+
+from anonymous_demo import __version__
+
+
+def save_args(config, save_path):
+    f = open(os.path.join(save_path), mode='w', encoding='utf8')
+    for arg in config.args:
+        if config.args_call_count[arg]:
+            f.write('{}: {}\n'.format(arg, config.args[arg]))
+    f.close()
+
+
+def print_args(config, logger=None, mode=0):
+    args = [key for key in sorted(config.args.keys())]
+    for arg in args:
+        if logger:
+            logger.info('{0}:{1}\t-->\tCalling Count:{2}'.format(arg, config.args[arg], config.args_call_count[arg]))
+        else:
+            print('{0}:{1}\t-->\tCalling Count:{2}'.format(arg, config.args[arg], config.args_call_count[arg]))
+
+
+def check_and_fix_labels(label_set: set, label_name, all_data, opt):
+    if '-100' in label_set:
+
+        label_to_index = {origin_label: int(idx) - 1 if origin_label != '-100' else -100 for origin_label, idx in zip(sorted(label_set), range(len(label_set)))}
+        index_to_label = {int(idx) - 1 if origin_label != '-100' else -100: origin_label for origin_label, idx in zip(sorted(label_set), range(len(label_set)))}
+    else:
+        label_to_index = {origin_label: int(idx) for origin_label, idx in zip(sorted(label_set), range(len(label_set)))}
+        index_to_label = {int(idx): origin_label for origin_label, idx in zip(sorted(label_set), range(len(label_set)))}
+    if 'index_to_label' not in opt.args:
+        opt.index_to_label = index_to_label
+        opt.label_to_index = label_to_index
+
+    if opt.index_to_label != index_to_label:
+        opt.index_to_label.update(index_to_label)
+        opt.label_to_index.update(label_to_index)
+    num_label = {l: 0 for l in label_set}
+    num_label['Sum'] = len(all_data)
+    for item in all_data:
+        try:
+            num_label[item[label_name]] += 1
+            item[label_name] = label_to_index[item[label_name]]
+        except Exception as e:
+            # print(e)
+            num_label[item.polarity] += 1
+            item.polarity = label_to_index[item.polarity]
+    print('Dataset Label Details: {}'.format(num_label))
+
+
+def check_and_fix_IOB_labels(label_map, opt):
+    index_to_IOB_label = {int(label_map[origin_label]): origin_label for origin_label in label_map}
+    opt.index_to_IOB_label = index_to_IOB_label
+
+
+def get_device(auto_device):
+    if isinstance(auto_device, str) and auto_device == 'allcuda':
+        device = 'cuda'
+    elif isinstance(auto_device, str):
+        device = auto_device
+    elif isinstance(auto_device, bool):
+        device = auto_cuda() if auto_device else 'cpu'
+    else:
+        device = auto_cuda()
+        try:
+            torch.device(device)
+        except RuntimeError as e:
+            print(colored('Device assignment error: {}, redirect to CPU'.format(e), 'red'))
+            device = 'cpu'
+    device_name = auto_cuda_name()
+    return device, device_name
+
+
+def _load_word_vec(path, word2idx=None, embed_dim=300):
+    fin = open(path, 'r', encoding='utf-8', newline='\n', errors='ignore')
+    word_vec = {}
+    for line in tqdm.tqdm(fin.readlines(), postfix='Loading embedding file...'):
+        tokens = line.rstrip().split()
+        word, vec = ' '.join(tokens[:-embed_dim]), tokens[-embed_dim:]
+        if word in word2idx.keys():
+            word_vec[word] = np.asarray(vec, dtype='float32')
+    return word_vec
+
+
+def build_embedding_matrix(word2idx, embed_dim, dat_fname, opt):
+    if not os.path.exists('run'):
+        os.makedirs('run')
+    embed_matrix_path = 'run/{}'.format(os.path.join(opt.dataset_name, dat_fname))
+    if os.path.exists(embed_matrix_path):
+        print(colored('Loading cached embedding_matrix from {} (Please remove all cached files if there is any problem!)'.format(embed_matrix_path), 'green'))
+        embedding_matrix = pickle.load(open(embed_matrix_path, 'rb'))
+    else:
+        glove_path = prepare_glove840_embedding(embed_matrix_path)
+        embedding_matrix = np.zeros((len(word2idx) + 2, embed_dim))
+
+        word_vec = _load_word_vec(glove_path, word2idx=word2idx, embed_dim=embed_dim)
+
+        for word, i in tqdm.tqdm(word2idx.items(), postfix=colored('Building embedding_matrix {}'.format(dat_fname), 'yellow')):
+            vec = word_vec.get(word)
+            if vec is not None:
+                embedding_matrix[i] = vec
+        pickle.dump(embedding_matrix, open(embed_matrix_path, 'wb'))
+    return embedding_matrix
+
+
+def pad_and_truncate(sequence, maxlen, dtype='int64', padding='post', truncating='post', value=0):
+    x = (np.ones(maxlen) * value).astype(dtype)
+    if truncating == 'pre':
+        trunc = sequence[-maxlen:]
+    else:
+        trunc = sequence[:maxlen]
+    trunc = np.asarray(trunc, dtype=dtype)
+    if padding == 'post':
+        x[:len(trunc)] = trunc
+    else:
+        x[-len(trunc):] = trunc
+    return x
+
+
+class TransformerConnectionError(ValueError):
+    def __init__(self):
+        pass
+
+
+def retry(f):
+    @wraps(f)
+    def decorated(*args, **kwargs):
+        count = 5
+        while count:
+
+            try:
+                return f(*args, **kwargs)
+            except (
+                TransformerConnectionError,
+                requests.exceptions.RequestException,
+                requests.exceptions.ConnectionError,
+                requests.exceptions.HTTPError,
+                requests.exceptions.ConnectTimeout,
+                requests.exceptions.ProxyError,
+                requests.exceptions.SSLError,
+                requests.exceptions.BaseHTTPError,
+            ) as e:
+                print(colored('Training Exception: {}, will retry later'.format(e)))
+                time.sleep(60)
+                count -= 1
+
+    return decorated
+
+
+def save_json(dic, save_path):
+    if isinstance(dic, str):
+        dic = eval(dic)
+    with open(save_path, 'w', encoding='utf-8') as f:
+        # f.write(str(dict))
+        str_ = json.dumps(dic, ensure_ascii=False)
+        f.write(str_)
+
+
+def load_json(save_path):
+    with open(save_path, 'r', encoding='utf-8') as f:
+        data = f.readline().strip()
+        print(type(data), data)
+        dic = json.loads(data)
+    return dic
+
+
+def init_optimizer(optimizer):
+    optimizers = {
+        'adadelta': torch.optim.Adadelta,  # default lr=1.0
+        'adagrad': torch.optim.Adagrad,  # default lr=0.01
+        'adam': torch.optim.Adam,  # default lr=0.001
+        'adamax': torch.optim.Adamax,  # default lr=0.002
+        'asgd': torch.optim.ASGD,  # default lr=0.01
+        'rmsprop': torch.optim.RMSprop,  # default lr=0.01
+        'sgd': torch.optim.SGD,
+        'adamw': torch.optim.AdamW,
+        torch.optim.Adadelta: torch.optim.Adadelta,  # default lr=1.0
+        torch.optim.Adagrad: torch.optim.Adagrad,  # default lr=0.01
+        torch.optim.Adam: torch.optim.Adam,  # default lr=0.001
+        torch.optim.Adamax: torch.optim.Adamax,  # default lr=0.002
+        torch.optim.ASGD: torch.optim.ASGD,  # default lr=0.01
+        torch.optim.RMSprop: torch.optim.RMSprop,  # default lr=0.01
+        torch.optim.SGD: torch.optim.SGD,
+        torch.optim.AdamW: torch.optim.AdamW,
+    }
+    if optimizer in optimizers:
+        return optimizers[optimizer]
+    elif hasattr(torch.optim, optimizer.__name__):
+        return optimizer
+    else:
+        raise KeyError('Unsupported optimizer: {}. Please use string or the optimizer objects in torch.optim as your optimizer'.format(optimizer))

anonymous_demo/utils/logger.py

@@ -0,0 +1,38 @@
+import logging
+import os
+import sys
+import time
+
+import termcolor
+
+today = time.strftime('%Y%m%d %H%M%S', time.localtime(time.time()))
+
+
+def get_logger(log_path, log_name='', log_type='training_log'):
+    if not log_path:
+        log_dir = os.path.join(log_path, "logs")
+    else:
+        log_dir = os.path.join('.', "logs")
+
+    full_path = os.path.join(log_dir, log_name + '_' + today)
+    if not os.path.exists(full_path):
+        os.makedirs(full_path)
+    log_path = os.path.join(full_path, "{}.log".format(log_type))
+    logger = logging.getLogger(log_name)
+    if not logger.handlers:
+        formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
+
+        file_handler = logging.FileHandler(log_path, encoding="utf8")
+        file_handler.setFormatter(formatter)
+        file_handler.setLevel(logging.INFO)
+
+        console_handler = logging.StreamHandler(sys.stdout)
+        console_handler.formatter = formatter
+        console_handler.setLevel(logging.INFO)
+
+        logger.addHandler(file_handler)
+        logger.addHandler(console_handler)
+
+        logger.setLevel(logging.INFO)
+
+    return logger

app.py

@@ -0,0 +1,271 @@
+import os
+import random
+import zipfile
+from difflib import Differ
+
+import gradio as gr
+import nltk
+import pandas as pd
+from findfile import find_files
+
+from anonymous_demo import TADCheckpointManager
+from textattack import Attacker
+from textattack.attack_recipes import BAEGarg2019, PWWSRen2019, TextFoolerJin2019, PSOZang2020, IGAWang2019, GeneticAlgorithmAlzantot2018, DeepWordBugGao2018
+from textattack.attack_results import SuccessfulAttackResult
+from textattack.datasets import Dataset
+from textattack.models.wrappers import HuggingFaceModelWrapper
+
+z = zipfile.ZipFile('checkpoints.zip', 'r')
+z.extractall(os.getcwd())
+
+class ModelWrapper(HuggingFaceModelWrapper):
+    def __init__(self, model):
+        self.model = model  # pipeline = pipeline
+
+    def __call__(self, text_inputs, **kwargs):
+        outputs = []
+        for text_input in text_inputs:
+            raw_outputs = self.model.infer(text_input, print_result=False, **kwargs)
+            outputs.append(raw_outputs['probs'])
+        return outputs
+
+
+class SentAttacker:
+
+    def __init__(self, model, recipe_class=BAEGarg2019):
+        model = model
+        model_wrapper = ModelWrapper(model)
+
+        recipe = recipe_class.build(model_wrapper)
+        # WordNet defaults to english. Set the default language to French ('fra')
+
+        # recipe.transformation.language = "en"
+
+        _dataset = [('', 0)]
+        _dataset = Dataset(_dataset)
+
+        self.attacker = Attacker(recipe, _dataset)
+
+
+def diff_texts(text1, text2):
+    d = Differ()
+    return [
+        (token[2:], token[0] if token[0] != " " else None)
+        for token in d.compare(text1, text2)
+    ]
+
+
+def get_ensembled_tad_results(results):
+    target_dict = {}
+    for r in results:
+        target_dict[r['label']] = target_dict.get(r['label']) + 1 if r['label'] in target_dict else 1
+
+    return dict(zip(target_dict.values(), target_dict.keys()))[max(target_dict.values())]
+
+
+nltk.download('omw-1.4')
+
+sent_attackers = {}
+tad_classifiers = {}
+
+attack_recipes = {
+    'bae': BAEGarg2019,
+    'pwws': PWWSRen2019,
+    'textfooler': TextFoolerJin2019,
+    'pso': PSOZang2020,
+    'iga': IGAWang2019,
+    'GA': GeneticAlgorithmAlzantot2018,
+    'wordbugger': DeepWordBugGao2018,
+}
+
+for attacker in [
+    'pwws',
+    'bae',
+    'textfooler'
+]:
+    for dataset in [
+        'agnews10k',
+        'amazon',
+        'sst2',
+    ]:
+        if 'tad-{}'.format(dataset) not in tad_classifiers:
+            tad_classifiers['tad-{}'.format(dataset)] = TADCheckpointManager.get_tad_text_classifier('tad-{}'.format(dataset).upper())
+
+        sent_attackers['tad-{}{}'.format(dataset, attacker)] = SentAttacker(tad_classifiers['tad-{}'.format(dataset)], attack_recipes[attacker])
+        tad_classifiers['tad-{}'.format(dataset)].sent_attacker = sent_attackers['tad-{}pwws'.format(dataset)]
+
+
+def get_a_sst2_example():
+    filter_key_words = ['.py', '.md', 'readme', 'log', 'result', 'zip', '.state_dict', '.model', '.png', 'acc_', 'f1_', '.origin', '.adv', '.csv']
+
+    dataset_file = {'train': [], 'test': [], 'valid': []}
+    dataset = 'sst2'
+    search_path = './'
+    task = 'text_defense'
+    dataset_file['test'] += find_files(search_path, [dataset, 'test', task], exclude_key=['.adv', '.org', '.defense', '.inference', 'train.'] + filter_key_words)
+
+    for dat_type in [
+        'test'
+    ]:
+        data = []
+        label_set = set()
+        for data_file in dataset_file[dat_type]:
+
+            with open(data_file, mode='r', encoding='utf8') as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    text, label = line.split('$LABEL$')
+                    text = text.strip()
+                    label = int(label.strip())
+                    data.append((text, label))
+                    label_set.add(label)
+        return data[random.randint(0, len(data))]
+
+
+def get_a_agnews_example():
+    filter_key_words = ['.py', '.md', 'readme', 'log', 'result', 'zip', '.state_dict', '.model', '.png', 'acc_', 'f1_', '.origin', '.adv', '.csv']
+
+    dataset_file = {'train': [], 'test': [], 'valid': []}
+    dataset = 'agnews'
+    search_path = './'
+    task = 'text_defense'
+    dataset_file['test'] += find_files(search_path, [dataset, 'test', task], exclude_key=['.adv', '.org', '.defense', '.inference', 'train.'] + filter_key_words)
+    for dat_type in [
+        'test'
+    ]:
+        data = []
+        label_set = set()
+        for data_file in dataset_file[dat_type]:
+
+            with open(data_file, mode='r', encoding='utf8') as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    text, label = line.split('$LABEL$')
+                    text = text.strip()
+                    label = int(label.strip())
+                    data.append((text, label))
+                    label_set.add(label)
+        return data[random.randint(0, len(data))]
+
+
+def get_a_amazon_example():
+    filter_key_words = ['.py', '.md', 'readme', 'log', 'result', 'zip', '.state_dict', '.model', '.png', 'acc_', 'f1_', '.origin', '.adv', '.csv']
+
+    dataset_file = {'train': [], 'test': [], 'valid': []}
+    dataset = 'amazon'
+    search_path = './'
+    task = 'text_defense'
+    dataset_file['test'] += find_files(search_path, [dataset, 'test', task], exclude_key=['.adv', '.org', '.defense', '.inference', 'train.'] + filter_key_words)
+
+    for dat_type in [
+        'test'
+    ]:
+        data = []
+        label_set = set()
+        for data_file in dataset_file[dat_type]:
+
+            with open(data_file, mode='r', encoding='utf8') as fin:
+                lines = fin.readlines()
+                for line in lines:
+                    text, label = line.split('$LABEL$')
+                    text = text.strip()
+                    label = int(label.strip())
+                    data.append((text, label))
+                    label_set.add(label)
+        return data[random.randint(0, len(data))]
+
+
+def generate_adversarial_example(dataset, attacker, text=None, label=None):
+    if not text:
+        if 'agnews' in dataset.lower():
+            text, label = get_a_agnews_example()
+        elif 'sst2' in dataset.lower():
+            text, label = get_a_sst2_example()
+        elif 'amazon' in dataset.lower():
+            text, label = get_a_amazon_example()
+
+    result = None
+    attack_result = sent_attackers['tad-{}{}'.format(dataset.lower(), attacker.lower())].attacker.simple_attack(text, int(label))
+    if isinstance(attack_result, SuccessfulAttackResult):
+
+        if (attack_result.perturbed_result.output != attack_result.original_result.ground_truth_output) and (attack_result.original_result.output == attack_result.original_result.ground_truth_output):
+            # with defense
+            result = tad_classifiers['tad-{}'.format(dataset.lower())].infer(
+                attack_result.perturbed_result.attacked_text.text + '!ref!{},{},{}'.format(attack_result.original_result.ground_truth_output, 1, attack_result.perturbed_result.output),
+                print_result=True,
+                defense='pwws',
+            )
+
+    if result:
+        classification_df = {}
+        classification_df['pred_label'] = result['label']
+        classification_df['confidence'] = round(result['confidence'], 3)
+        classification_df['is_correct'] = result['ref_label_check']
+        classification_df['is_repaired'] = result['is_fixed']
+
+        advdetection_df = {}
+        if result['is_adv_label'] != '0':
+            advdetection_df['is_adversary'] = result['is_adv_label']
+            advdetection_df['perturbed_label'] = result['perturbed_label']
+            advdetection_df['confidence'] = round(result['is_adv_confidence'], 3)
+            # advdetection_df['ref_is_attack'] = result['ref_is_adv_label']
+            # advdetection_df['is_correct'] = result['ref_is_adv_check']
+
+    else:
+        return generate_adversarial_example(dataset, attacker)
+
+    return (text,
+            label,
+            attack_result.perturbed_result.attacked_text.text,
+            diff_texts(text, attack_result.perturbed_result.attacked_text.text),
+            diff_texts(text, result['restored_text']),
+            attack_result.perturbed_result.output,
+            pd.DataFrame(classification_df, index=[0]),
+            pd.DataFrame(advdetection_df, index=[0])
+            )
+
+
+demo = gr.Blocks()
+
+with demo:
+    with gr.Row():
+        with gr.Column():
+            input_dataset = gr.Radio(choices=['SST2', 'AGNews10K', 'Amazon'], value='Amazon', label="Dataset")
+            input_attacker = gr.Radio(choices=['BAE', 'PWWS', 'TextFooler'], value='TextFooler', label="Attacker")
+            input_sentence = gr.Textbox(placeholder='Randomly choose a example from testing set if this box is blank', label="Sentence")
+            input_label = gr.Textbox(placeholder='original label ... ', label="Original Label")
+
+            gr.Markdown("Original Example")
+
+            output_origin_example = gr.Textbox(label="Original Example")
+            output_original_label = gr.Textbox(label="Original Label")
+
+            gr.Markdown("Adversarial Example")
+            output_adv_example = gr.Textbox(label="Adversarial Example")
+            output_adv_label = gr.Textbox(label="Perturbed Label")
+
+            gr.Markdown('This demo is deployed on a CPU device so it may take a long time to execute. Please be patient.')
+            button_gen = gr.Button("Click Here to Generate an Adversary and Run Adversary Detection & Repair")
+
+        # Right column (outputs)
+        with gr.Column():
+            gr.Markdown("Example Difference")
+            adv_text_diff = gr.HighlightedText(label="Adversarial Example Difference", combine_adjacent=True)
+            restored_text_diff = gr.HighlightedText(label="Restored Example Difference", combine_adjacent=True)
+
+            output_is_adv_df = gr.DataFrame(label="Adversary Prediction")
+            output_df = gr.DataFrame(label="Standard Classification Prediction")
+
+    # Bind functions to buttons
+    button_gen.click(fn=generate_adversarial_example,
+                     inputs=[input_dataset, input_attacker, input_sentence, input_label],
+                     outputs=[output_origin_example,
+                              output_original_label,
+                              output_adv_example,
+                              adv_text_diff,
+                              restored_text_diff,
+                              output_adv_label,
+                              output_df,
+                              output_is_adv_df])
+
+demo.launch()

checkpoints.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a5452cd89dcd3132d616cc81e2a1b063efa7d11e5798719b0779715b1c6edeb
+size 1846862527

requirements.txt

@@ -0,0 +1,19 @@
+findfile>=1.7.9.8
+autocuda>=0.11
+metric-visualizer>=0.5.5
+boostaug>=2.2.3
+spacy
+networkx
+seqeval
+update-checker
+typing_extensions
+tqdm
+pytorch_warmup
+termcolor
+gitpython
+gdown>=4.4.0
+transformers>4.20.0
+torch>1.0.0
+sentencepiece
+tensorflow_text
+textattack

textattack/__init__.py

@@ -0,0 +1,39 @@
+"""Welcome to the API references for TextAttack!
+
+What is TextAttack?
+
+`TextAttack <https://github.com/QData/TextAttack>`__ is a Python framework for adversarial attacks, adversarial training, and data augmentation in NLP.
+
+TextAttack makes experimenting with the robustness of NLP models seamless, fast, and easy. It's also useful for NLP model training, adversarial training, and data augmentation.
+
+TextAttack provides components for common NLP tasks like sentence encoding, grammar-checking, and word replacement that can be used on their own.
+"""
+from .attack_args import AttackArgs, CommandLineAttackArgs
+from .augment_args import AugmenterArgs
+from .dataset_args import DatasetArgs
+from .model_args import ModelArgs
+from .training_args import TrainingArgs, CommandLineTrainingArgs
+from .attack import Attack
+from .attacker import Attacker
+from .trainer import Trainer
+from .metrics import Metric
+
+from . import (
+    attack_recipes,
+    attack_results,
+    augmentation,
+    commands,
+    constraints,
+    datasets,
+    goal_function_results,
+    goal_functions,
+    loggers,
+    metrics,
+    models,
+    search_methods,
+    shared,
+    transformations,
+)
+
+
+name = "textattack"

textattack/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python
+
+if __name__ == "__main__":
+    import textattack
+
+    textattack.commands.textattack_cli.main()

textattack/attack.py

@@ -0,0 +1,492 @@
+"""
+Attack Class
+============
+"""
+
+from collections import OrderedDict
+from typing import List, Union
+
+import lru
+import torch
+
+import textattack
+from textattack.attack_results import (
+    FailedAttackResult,
+    MaximizedAttackResult,
+    SkippedAttackResult,
+    SuccessfulAttackResult,
+)
+from textattack.constraints import Constraint, PreTransformationConstraint
+from textattack.goal_function_results import GoalFunctionResultStatus
+from textattack.goal_functions import GoalFunction
+from textattack.models.wrappers import ModelWrapper
+from textattack.search_methods import SearchMethod
+from textattack.shared import AttackedText, utils
+from textattack.transformations import CompositeTransformation, Transformation
+
+
+class Attack:
+    """An attack generates adversarial examples on text.
+
+    An attack is comprised of a goal function, constraints, transformation, and a search method. Use :meth:`attack` method to attack one sample at a time.
+
+    Args:
+        goal_function (:class:`~textattack.goal_functions.GoalFunction`):
+            A function for determining how well a perturbation is doing at achieving the attack's goal.
+        constraints (list of :class:`~textattack.constraints.Constraint` or :class:`~textattack.constraints.PreTransformationConstraint`):
+            A list of constraints to add to the attack, defining which perturbations are valid.
+        transformation (:class:`~textattack.transformations.Transformation`):
+            The transformation applied at each step of the attack.
+        search_method (:class:`~textattack.search_methods.SearchMethod`):
+            The method for exploring the search space of possible perturbations
+        transformation_cache_size (:obj:`int`, `optional`, defaults to :obj:`2**15`):
+            The number of items to keep in the transformations cache
+        constraint_cache_size (:obj:`int`, `optional`, defaults to :obj:`2**15`):
+            The number of items to keep in the constraints cache
+
+    Example::
+
+        >>> import textattack
+        >>> import transformers
+
+        >>> # Load model, tokenizer, and model_wrapper
+        >>> model = transformers.AutoModelForSequenceClassification.from_pretrained("textattack/bert-base-uncased-imdb")
+        >>> tokenizer = transformers.AutoTokenizer.from_pretrained("textattack/bert-base-uncased-imdb")
+        >>> model_wrapper = textattack.models.wrappers.HuggingFaceModelWrapper(model, tokenizer)
+
+        >>> # Construct our four components for `Attack`
+        >>> from textattack.constraints.pre_transformation import RepeatModification, StopwordModification
+        >>> from textattack.constraints.semantics import WordEmbeddingDistance
+
+        >>> goal_function = textattack.goal_functions.UntargetedClassification(model_wrapper)
+        >>> constraints = [
+        ...     RepeatModification(),
+        ...     StopwordModification()
+        ...     WordEmbeddingDistance(min_cos_sim=0.9)
+        ... ]
+        >>> transformation = WordSwapEmbedding(max_candidates=50)
+        >>> search_method = GreedyWordSwapWIR(wir_method="delete")
+
+        >>> # Construct the actual attack
+        >>> attack = Attack(goal_function, constraints, transformation, search_method)
+
+        >>> input_text = "I really enjoyed the new movie that came out last month."
+        >>> label = 1 #Positive
+        >>> attack_result = attack.attack(input_text, label)
+    """
+
+    def __init__(
+        self,
+        goal_function: GoalFunction,
+        constraints: List[Union[Constraint, PreTransformationConstraint]],
+        transformation: Transformation,
+        search_method: SearchMethod,
+        transformation_cache_size=2**15,
+        constraint_cache_size=2**15,
+    ):
+        """Initialize an attack object.
+
+        Attacks can be run multiple times.
+        """
+        assert isinstance(
+            goal_function, GoalFunction
+        ), f"`goal_function` must be of type `textattack.goal_functions.GoalFunction`, but got type `{type(goal_function)}`."
+        assert isinstance(
+            constraints, list
+        ), "`constraints` must be a list of `textattack.constraints.Constraint` or `textattack.constraints.PreTransformationConstraint`."
+        for c in constraints:
+            assert isinstance(
+                c, (Constraint, PreTransformationConstraint)
+            ), "`constraints` must be a list of `textattack.constraints.Constraint` or `textattack.constraints.PreTransformationConstraint`."
+        assert isinstance(
+            transformation, Transformation
+        ), f"`transformation` must be of type `textattack.transformations.Transformation`, but got type `{type(transformation)}`."
+        assert isinstance(
+            search_method, SearchMethod
+        ), f"`search_method` must be of type `textattack.search_methods.SearchMethod`, but got type `{type(search_method)}`."
+
+        self.goal_function = goal_function
+        self.search_method = search_method
+        self.transformation = transformation
+        self.is_black_box = (
+            getattr(transformation, "is_black_box", True) and search_method.is_black_box
+        )
+
+        if not self.search_method.check_transformation_compatibility(
+            self.transformation
+        ):
+            raise ValueError(
+                f"SearchMethod {self.search_method} incompatible with transformation {self.transformation}"
+            )
+
+        self.constraints = []
+        self.pre_transformation_constraints = []
+        for constraint in constraints:
+            if isinstance(
+                constraint,
+                textattack.constraints.PreTransformationConstraint,
+            ):
+                self.pre_transformation_constraints.append(constraint)
+            else:
+                self.constraints.append(constraint)
+
+        # Check if we can use transformation cache for our transformation.
+        if not self.transformation.deterministic:
+            self.use_transformation_cache = False
+        elif isinstance(self.transformation, CompositeTransformation):
+            self.use_transformation_cache = True
+            for t in self.transformation.transformations:
+                if not t.deterministic:
+                    self.use_transformation_cache = False
+                    break
+        else:
+            self.use_transformation_cache = True
+        self.transformation_cache_size = transformation_cache_size
+        self.transformation_cache = lru.LRU(transformation_cache_size)
+
+        self.constraint_cache_size = constraint_cache_size
+        self.constraints_cache = lru.LRU(constraint_cache_size)
+
+        # Give search method access to functions for getting transformations and evaluating them
+        self.search_method.get_transformations = self.get_transformations
+        # Give search method access to self.goal_function for model query count, etc.
+        self.search_method.goal_function = self.goal_function
+        # The search method only needs access to the first argument. The second is only used
+        # by the attack class when checking whether to skip the sample
+        self.search_method.get_goal_results = self.goal_function.get_results
+
+        # Give search method access to get indices which need to be ordered / searched
+        self.search_method.get_indices_to_order = self.get_indices_to_order
+
+        self.search_method.filter_transformations = self.filter_transformations
+
+    def clear_cache(self, recursive=True):
+        self.constraints_cache.clear()
+        if self.use_transformation_cache:
+            self.transformation_cache.clear()
+        if recursive:
+            self.goal_function.clear_cache()
+            for constraint in self.constraints:
+                if hasattr(constraint, "clear_cache"):
+                    constraint.clear_cache()
+
+    def cpu_(self):
+        """Move any `torch.nn.Module` models that are part of Attack to CPU."""
+        visited = set()
+
+        def to_cpu(obj):
+            visited.add(id(obj))
+            if isinstance(obj, torch.nn.Module):
+                obj.cpu()
+            elif isinstance(
+                obj,
+                (
+                    Attack,
+                    GoalFunction,
+                    Transformation,
+                    SearchMethod,
+                    Constraint,
+                    PreTransformationConstraint,
+                    ModelWrapper,
+                ),
+            ):
+                for key in obj.__dict__:
+                    s_obj = obj.__dict__[key]
+                    if id(s_obj) not in visited:
+                        to_cpu(s_obj)
+            elif isinstance(obj, (list, tuple)):
+                for item in obj:
+                    if id(item) not in visited and isinstance(
+                        item, (Transformation, Constraint, PreTransformationConstraint)
+                    ):
+                        to_cpu(item)
+
+        to_cpu(self)
+
+    def cuda_(self):
+        """Move any `torch.nn.Module` models that are part of Attack to GPU."""
+        visited = set()
+
+        def to_cuda(obj):
+            visited.add(id(obj))
+            if isinstance(obj, torch.nn.Module):
+                obj.to(textattack.shared.utils.device)
+            elif isinstance(
+                obj,
+                (
+                    Attack,
+                    GoalFunction,
+                    Transformation,
+                    SearchMethod,
+                    Constraint,
+                    PreTransformationConstraint,
+                    ModelWrapper,
+                ),
+            ):
+                for key in obj.__dict__:
+                    s_obj = obj.__dict__[key]
+                    if id(s_obj) not in visited:
+                        to_cuda(s_obj)
+            elif isinstance(obj, (list, tuple)):
+                for item in obj:
+                    if id(item) not in visited and isinstance(
+                        item, (Transformation, Constraint, PreTransformationConstraint)
+                    ):
+                        to_cuda(item)
+
+        to_cuda(self)
+
+    def get_indices_to_order(self, current_text, **kwargs):
+        """Applies ``pre_transformation_constraints`` to ``text`` to get all
+        the indices that can be used to search and order.
+
+        Args:
+            current_text: The current ``AttackedText`` for which we need to find indices are eligible to be ordered.
+        Returns:
+            The length and the filtered list of indices which search methods can use to search/order.
+        """
+
+        indices_to_order = self.transformation(
+            current_text,
+            pre_transformation_constraints=self.pre_transformation_constraints,
+            return_indices=True,
+            **kwargs,
+        )
+
+        len_text = len(indices_to_order)
+
+        # Convert indices_to_order to list for easier shuffling later
+        return len_text, list(indices_to_order)
+
+    def _get_transformations_uncached(self, current_text, original_text=None, **kwargs):
+        """Applies ``self.transformation`` to ``text``, then filters the list
+        of possible transformations through the applicable constraints.
+
+        Args:
+            current_text: The current ``AttackedText`` on which to perform the transformations.
+            original_text: The original ``AttackedText`` from which the attack started.
+        Returns:
+            A filtered list of transformations where each transformation matches the constraints
+        """
+        transformed_texts = self.transformation(
+            current_text,
+            pre_transformation_constraints=self.pre_transformation_constraints,
+            **kwargs,
+        )
+
+        return transformed_texts
+
+    def get_transformations(self, current_text, original_text=None, **kwargs):
+        """Applies ``self.transformation`` to ``text``, then filters the list
+        of possible transformations through the applicable constraints.
+
+        Args:
+            current_text: The current ``AttackedText`` on which to perform the transformations.
+            original_text: The original ``AttackedText`` from which the attack started.
+        Returns:
+            A filtered list of transformations where each transformation matches the constraints
+        """
+        if not self.transformation:
+            raise RuntimeError(
+                "Cannot call `get_transformations` without a transformation."
+            )
+
+        if self.use_transformation_cache:
+            cache_key = tuple([current_text] + sorted(kwargs.items()))
+            if utils.hashable(cache_key) and cache_key in self.transformation_cache:
+                # promote transformed_text to the top of the LRU cache
+                self.transformation_cache[cache_key] = self.transformation_cache[
+                    cache_key
+                ]
+                transformed_texts = list(self.transformation_cache[cache_key])
+            else:
+                transformed_texts = self._get_transformations_uncached(
+                    current_text, original_text, **kwargs
+                )
+                if utils.hashable(cache_key):
+                    self.transformation_cache[cache_key] = tuple(transformed_texts)
+        else:
+            transformed_texts = self._get_transformations_uncached(
+                current_text, original_text, **kwargs
+            )
+
+        return self.filter_transformations(
+            transformed_texts, current_text, original_text
+        )
+
+    def _filter_transformations_uncached(
+        self, transformed_texts, current_text, original_text=None
+    ):
+        """Filters a list of potential transformed texts based on
+        ``self.constraints``
+
+        Args:
+            transformed_texts: A list of candidate transformed ``AttackedText`` to filter.
+            current_text: The current ``AttackedText`` on which the transformation was applied.
+            original_text: The original ``AttackedText`` from which the attack started.
+        """
+        filtered_texts = transformed_texts[:]
+        for C in self.constraints:
+            if len(filtered_texts) == 0:
+                break
+            if C.compare_against_original:
+                if not original_text:
+                    raise ValueError(
+                        f"Missing `original_text` argument when constraint {type(C)} is set to compare against `original_text`"
+                    )
+
+                filtered_texts = C.call_many(filtered_texts, original_text)
+            else:
+                filtered_texts = C.call_many(filtered_texts, current_text)
+        # Default to false for all original transformations.
+        for original_transformed_text in transformed_texts:
+            self.constraints_cache[(current_text, original_transformed_text)] = False
+        # Set unfiltered transformations to True in the cache.
+        for filtered_text in filtered_texts:
+            self.constraints_cache[(current_text, filtered_text)] = True
+        return filtered_texts
+
+    def filter_transformations(
+        self, transformed_texts, current_text, original_text=None
+    ):
+        """Filters a list of potential transformed texts based on
+        ``self.constraints`` Utilizes an LRU cache to attempt to avoid
+        recomputing common transformations.
+
+        Args:
+            transformed_texts: A list of candidate transformed ``AttackedText`` to filter.
+            current_text: The current ``AttackedText`` on which the transformation was applied.
+            original_text: The original ``AttackedText`` from which the attack started.
+        """
+        # Remove any occurences of current_text in transformed_texts
+        transformed_texts = [
+            t for t in transformed_texts if t.text != current_text.text
+        ]
+        # Populate cache with transformed_texts
+        uncached_texts = []
+        filtered_texts = []
+        for transformed_text in transformed_texts:
+            if (current_text, transformed_text) not in self.constraints_cache:
+                uncached_texts.append(transformed_text)
+            else:
+                # promote transformed_text to the top of the LRU cache
+                self.constraints_cache[
+                    (current_text, transformed_text)
+                ] = self.constraints_cache[(current_text, transformed_text)]
+                if self.constraints_cache[(current_text, transformed_text)]:
+                    filtered_texts.append(transformed_text)
+        filtered_texts += self._filter_transformations_uncached(
+            uncached_texts, current_text, original_text=original_text
+        )
+        # Sort transformations to ensure order is preserved between runs
+        filtered_texts.sort(key=lambda t: t.text)
+        return filtered_texts
+
+    def _attack(self, initial_result):
+        """Calls the ``SearchMethod`` to perturb the ``AttackedText`` stored in
+        ``initial_result``.
+
+        Args:
+            initial_result: The initial ``GoalFunctionResult`` from which to perturb.
+
+        Returns:
+            A ``SuccessfulAttackResult``, ``FailedAttackResult``,
+                or ``MaximizedAttackResult``.
+        """
+        final_result = self.search_method(initial_result)
+        self.clear_cache()
+        if final_result.goal_status == GoalFunctionResultStatus.SUCCEEDED:
+            result = SuccessfulAttackResult(
+                initial_result,
+                final_result,
+            )
+        elif final_result.goal_status == GoalFunctionResultStatus.SEARCHING:
+            result = FailedAttackResult(
+                initial_result,
+                final_result,
+            )
+        elif final_result.goal_status == GoalFunctionResultStatus.MAXIMIZING:
+            result = MaximizedAttackResult(
+                initial_result,
+                final_result,
+            )
+        else:
+            raise ValueError(f"Unrecognized goal status {final_result.goal_status}")
+        return result
+
+    def attack(self, example, ground_truth_output):
+        """Attack a single example.
+
+        Args:
+            example (:obj:`str`, :obj:`OrderedDict[str, str]` or :class:`~textattack.shared.AttackedText`):
+                Example to attack. It can be a single string or an `OrderedDict` where
+                keys represent the input fields (e.g. "premise", "hypothesis") and the values are the actual input textx.
+                Also accepts :class:`~textattack.shared.AttackedText` that wraps around the input.
+            ground_truth_output(:obj:`int`, :obj:`float` or :obj:`str`):
+                Ground truth output of `example`.
+                For classification tasks, it should be an integer representing the ground truth label.
+                For regression tasks (e.g. STS), it should be the target value.
+                For seq2seq tasks (e.g. translation), it should be the target string.
+        Returns:
+            :class:`~textattack.attack_results.AttackResult` that represents the result of the attack.
+        """
+        assert isinstance(
+            example, (str, OrderedDict, AttackedText)
+        ), "`example` must either be `str`, `collections.OrderedDict`, `textattack.shared.AttackedText`."
+        if isinstance(example, (str, OrderedDict)):
+            example = AttackedText(example)
+
+        assert isinstance(
+            ground_truth_output, (int, str)
+        ), "`ground_truth_output` must either be `str` or `int`."
+        goal_function_result, _ = self.goal_function.init_attack_example(
+            example, ground_truth_output
+        )
+        if goal_function_result.goal_status == GoalFunctionResultStatus.SKIPPED:
+            return SkippedAttackResult(goal_function_result)
+        else:
+            result = self._attack(goal_function_result)
+            return result
+
+    def __repr__(self):
+        """Prints attack parameters in a human-readable string.
+
+        Inspired by the readability of printing PyTorch nn.Modules:
+        https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/module.py
+        """
+        main_str = "Attack" + "("
+        lines = []
+
+        lines.append(utils.add_indent(f"(search_method): {self.search_method}", 2))
+        # self.goal_function
+        lines.append(utils.add_indent(f"(goal_function):  {self.goal_function}", 2))
+        # self.transformation
+        lines.append(utils.add_indent(f"(transformation):  {self.transformation}", 2))
+        # self.constraints
+        constraints_lines = []
+        constraints = self.constraints + self.pre_transformation_constraints
+        if len(constraints):
+            for i, constraint in enumerate(constraints):
+                constraints_lines.append(utils.add_indent(f"({i}): {constraint}", 2))
+            constraints_str = utils.add_indent("\n" + "\n".join(constraints_lines), 2)
+        else:
+            constraints_str = "None"
+        lines.append(utils.add_indent(f"(constraints): {constraints_str}", 2))
+        # self.is_black_box
+        lines.append(utils.add_indent(f"(is_black_box):  {self.is_black_box}", 2))
+        main_str += "\n  " + "\n  ".join(lines) + "\n"
+        main_str += ")"
+        return main_str
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["transformation_cache"] = None
+        state["constraints_cache"] = None
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__ = state
+        self.transformation_cache = lru.LRU(self.transformation_cache_size)
+        self.constraints_cache = lru.LRU(self.constraint_cache_size)
+
+    __str__ = __repr__

textattack/attack_args.py

@@ -0,0 +1,763 @@
+"""
+AttackArgs Class
+================
+"""
+
+from dataclasses import dataclass, field
+import json
+import os
+import sys
+import time
+from typing import Dict, Optional
+
+import textattack
+from textattack.shared.utils import ARGS_SPLIT_TOKEN, load_module_from_file
+
+from .attack import Attack
+from .dataset_args import DatasetArgs
+from .model_args import ModelArgs
+
+ATTACK_RECIPE_NAMES = {
+    "alzantot": "textattack.attack_recipes.GeneticAlgorithmAlzantot2018",
+    "bae": "textattack.attack_recipes.BAEGarg2019",
+    "bert-attack": "textattack.attack_recipes.BERTAttackLi2020",
+    "faster-alzantot": "textattack.attack_recipes.FasterGeneticAlgorithmJia2019",
+    "deepwordbug": "textattack.attack_recipes.DeepWordBugGao2018",
+    "hotflip": "textattack.attack_recipes.HotFlipEbrahimi2017",
+    "input-reduction": "textattack.attack_recipes.InputReductionFeng2018",
+    "kuleshov": "textattack.attack_recipes.Kuleshov2017",
+    "morpheus": "textattack.attack_recipes.MorpheusTan2020",
+    "seq2sick": "textattack.attack_recipes.Seq2SickCheng2018BlackBox",
+    "textbugger": "textattack.attack_recipes.TextBuggerLi2018",
+    "textfooler": "textattack.attack_recipes.TextFoolerJin2019",
+    "pwws": "textattack.attack_recipes.PWWSRen2019",
+    "iga": "textattack.attack_recipes.IGAWang2019",
+    "pruthi": "textattack.attack_recipes.Pruthi2019",
+    "pso": "textattack.attack_recipes.PSOZang2020",
+    "checklist": "textattack.attack_recipes.CheckList2020",
+    "clare": "textattack.attack_recipes.CLARE2020",
+    "a2t": "textattack.attack_recipes.A2TYoo2021",
+}
+
+
+BLACK_BOX_TRANSFORMATION_CLASS_NAMES = {
+    "random-synonym-insertion": "textattack.transformations.RandomSynonymInsertion",
+    "word-deletion": "textattack.transformations.WordDeletion",
+    "word-swap-embedding": "textattack.transformations.WordSwapEmbedding",
+    "word-swap-homoglyph": "textattack.transformations.WordSwapHomoglyphSwap",
+    "word-swap-inflections": "textattack.transformations.WordSwapInflections",
+    "word-swap-neighboring-char-swap": "textattack.transformations.WordSwapNeighboringCharacterSwap",
+    "word-swap-random-char-deletion": "textattack.transformations.WordSwapRandomCharacterDeletion",
+    "word-swap-random-char-insertion": "textattack.transformations.WordSwapRandomCharacterInsertion",
+    "word-swap-random-char-substitution": "textattack.transformations.WordSwapRandomCharacterSubstitution",
+    "word-swap-wordnet": "textattack.transformations.WordSwapWordNet",
+    "word-swap-masked-lm": "textattack.transformations.WordSwapMaskedLM",
+    "word-swap-hownet": "textattack.transformations.WordSwapHowNet",
+    "word-swap-qwerty": "textattack.transformations.WordSwapQWERTY",
+}
+
+
+WHITE_BOX_TRANSFORMATION_CLASS_NAMES = {
+    "word-swap-gradient": "textattack.transformations.WordSwapGradientBased"
+}
+
+
+CONSTRAINT_CLASS_NAMES = {
+    #
+    # Semantics constraints
+    #
+    "embedding": "textattack.constraints.semantics.WordEmbeddingDistance",
+    "bert": "textattack.constraints.semantics.sentence_encoders.BERT",
+    "infer-sent": "textattack.constraints.semantics.sentence_encoders.InferSent",
+    "thought-vector": "textattack.constraints.semantics.sentence_encoders.ThoughtVector",
+    "use": "textattack.constraints.semantics.sentence_encoders.UniversalSentenceEncoder",
+    "muse": "textattack.constraints.semantics.sentence_encoders.MultilingualUniversalSentenceEncoder",
+    "bert-score": "textattack.constraints.semantics.BERTScore",
+    #
+    # Grammaticality constraints
+    #
+    "lang-tool": "textattack.constraints.grammaticality.LanguageTool",
+    "part-of-speech": "textattack.constraints.grammaticality.PartOfSpeech",
+    "goog-lm": "textattack.constraints.grammaticality.language_models.GoogleLanguageModel",
+    "gpt2": "textattack.constraints.grammaticality.language_models.GPT2",
+    "learning-to-write": "textattack.constraints.grammaticality.language_models.LearningToWriteLanguageModel",
+    "cola": "textattack.constraints.grammaticality.COLA",
+    #
+    # Overlap constraints
+    #
+    "bleu": "textattack.constraints.overlap.BLEU",
+    "chrf": "textattack.constraints.overlap.chrF",
+    "edit-distance": "textattack.constraints.overlap.LevenshteinEditDistance",
+    "meteor": "textattack.constraints.overlap.METEOR",
+    "max-words-perturbed": "textattack.constraints.overlap.MaxWordsPerturbed",
+    #
+    # Pre-transformation constraints
+    #
+    "repeat": "textattack.constraints.pre_transformation.RepeatModification",
+    "stopword": "textattack.constraints.pre_transformation.StopwordModification",
+    "max-word-index": "textattack.constraints.pre_transformation.MaxWordIndexModification",
+}
+
+
+SEARCH_METHOD_CLASS_NAMES = {
+    "beam-search": "textattack.search_methods.BeamSearch",
+    "greedy": "textattack.search_methods.GreedySearch",
+    "ga-word": "textattack.search_methods.GeneticAlgorithm",
+    "greedy-word-wir": "textattack.search_methods.GreedyWordSwapWIR",
+    "pso": "textattack.search_methods.ParticleSwarmOptimization",
+}
+
+
+GOAL_FUNCTION_CLASS_NAMES = {
+    #
+    # Classification goal functions
+    #
+    "targeted-classification": "textattack.goal_functions.classification.TargetedClassification",
+    "untargeted-classification": "textattack.goal_functions.classification.UntargetedClassification",
+    "input-reduction": "textattack.goal_functions.classification.InputReduction",
+    #
+    # Text goal functions
+    #
+    "minimize-bleu": "textattack.goal_functions.text.MinimizeBleu",
+    "non-overlapping-output": "textattack.goal_functions.text.NonOverlappingOutput",
+    "text-to-text": "textattack.goal_functions.text.TextToTextGoalFunction",
+}
+
+
+@dataclass
+class AttackArgs:
+    """Attack arguments to be passed to :class:`~textattack.Attacker`.
+
+    Args:
+        num_examples (:obj:`int`, 'optional`, defaults to :obj:`10`):
+            The number of examples to attack. :obj:`-1` for entire dataset.
+        num_successful_examples (:obj:`int`, `optional`, defaults to :obj:`None`):
+            The number of successful adversarial examples we want. This is different from :obj:`num_examples`
+            as :obj:`num_examples` only cares about attacking `N` samples while :obj:`num_successful_examples` aims to keep attacking
+            until we have `N` successful cases.
+
+            .. note::
+                If set, this argument overrides `num_examples` argument.
+        num_examples_offset (:obj: `int`, `optional`, defaults to :obj:`0`):
+            The offset index to start at in the dataset.
+        attack_n (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Whether to run attack until total of `N` examples have been attacked (and not skipped).
+        shuffle (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If :obj:`True`, we randomly shuffle the dataset before attacking. However, this avoids actually shuffling
+            the dataset internally and opts for shuffling the list of indices of examples we want to attack. This means
+            :obj:`shuffle` can now be used with checkpoint saving.
+        query_budget (:obj:`int`, `optional`, defaults to :obj:`None`):
+            The maximum number of model queries allowed per example attacked.
+            If not set, we use the query budget set in the :class:`~textattack.goal_functions.GoalFunction` object (which by default is :obj:`float("inf")`).
+
+            .. note::
+                Setting this overwrites the query budget set in :class:`~textattack.goal_functions.GoalFunction` object.
+        checkpoint_interval (:obj:`int`, `optional`, defaults to :obj:`None`):
+            If set, checkpoint will be saved after attacking every `N` examples. If :obj:`None` is passed, no checkpoints will be saved.
+        checkpoint_dir (:obj:`str`, `optional`, defaults to :obj:`"checkpoints"`):
+            The directory to save checkpoint files.
+        random_seed (:obj:`int`, `optional`, defaults to :obj:`765`):
+            Random seed for reproducibility.
+        parallel (:obj:`False`, `optional`, defaults to :obj:`False`):
+            If :obj:`True`, run attack using multiple CPUs/GPUs.
+        num_workers_per_device (:obj:`int`, `optional`, defaults to :obj:`1`):
+            Number of worker processes to run per device in parallel mode (i.e. :obj:`parallel=True`). For example, if you are using GPUs and :obj:`num_workers_per_device=2`,
+            then 2 processes will be running in each GPU.
+        log_to_txt (:obj:`str`, `optional`, defaults to :obj:`None`):
+            If set, save attack logs as a `.txt` file to the directory specified by this argument.
+            If the last part of the provided path ends with `.txt` extension, it is assumed to the desired path of the log file.
+        log_to_csv (:obj:`str`, `optional`, defaults to :obj:`None`):
+            If set, save attack logs as a CSV file to the directory specified by this argument.
+            If the last part of the provided path ends with `.csv` extension, it is assumed to the desired path of the log file.
+        csv_coloring_style (:obj:`str`, `optional`, defaults to :obj:`"file"`):
+            Method for choosing how to mark perturbed parts of the text. Options are :obj:`"file"`, :obj:`"plain"`, and :obj:`"html"`.
+            :obj:`"file"` wraps perturbed parts with double brackets :obj:`[[ <text> ]]` while :obj:`"plain"` does not mark the text in any way.
+        log_to_visdom (:obj:`dict`, `optional`, defaults to :obj:`None`):
+            If set, Visdom logger is used with the provided dictionary passed as a keyword arguments to :class:`~textattack.loggers.VisdomLogger`.
+            Pass in empty dictionary to use default arguments. For custom logger, the dictionary should have the following
+            three keys and their corresponding values: :obj:`"env", "port", "hostname"`.
+        log_to_wandb(:obj:`dict`, `optional`, defaults to :obj:`None`):
+            If set, WandB logger is used with the provided dictionary passed as a keyword arguments to :class:`~textattack.loggers.WeightsAndBiasesLogger`.
+            Pass in empty dictionary to use default arguments. For custom logger, the dictionary should have the following
+            key and its corresponding value: :obj:`"project"`.
+        disable_stdout (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Disable displaying individual attack results to stdout.
+        silent (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Disable all logging (except for errors). This is stronger than :obj:`disable_stdout`.
+        enable_advance_metrics (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            Enable calculation and display of optional advance post-hoc metrics like perplexity, grammar errors, etc.
+    """
+
+    num_examples: int = 10
+    num_successful_examples: int = None
+    num_examples_offset: int = 0
+    attack_n: bool = False
+    shuffle: bool = False
+    query_budget: int = None
+    checkpoint_interval: int = None
+    checkpoint_dir: str = "checkpoints"
+    random_seed: int = 765  # equivalent to sum((ord(c) for c in "TEXTATTACK"))
+    parallel: bool = False
+    num_workers_per_device: int = 1
+    log_to_txt: str = None
+    log_to_csv: str = None
+    log_summary_to_json: str = None
+    csv_coloring_style: str = "file"
+    log_to_visdom: dict = None
+    log_to_wandb: dict = None
+    disable_stdout: bool = False
+    silent: bool = False
+    enable_advance_metrics: bool = False
+    metrics: Optional[Dict] = None
+
+    def __post_init__(self):
+        if self.num_successful_examples:
+            self.num_examples = None
+        if self.num_examples:
+            assert (
+                self.num_examples >= 0 or self.num_examples == -1
+            ), "`num_examples` must be greater than or equal to 0 or equal to -1."
+        if self.num_successful_examples:
+            assert (
+                self.num_successful_examples >= 0
+            ), "`num_examples` must be greater than or equal to 0."
+
+        if self.query_budget:
+            assert self.query_budget > 0, "`query_budget` must be greater than 0."
+
+        if self.checkpoint_interval:
+            assert (
+                self.checkpoint_interval > 0
+            ), "`checkpoint_interval` must be greater than 0."
+
+        assert (
+            self.num_workers_per_device > 0
+        ), "`num_workers_per_device` must be greater than 0."
+
+    @classmethod
+    def _add_parser_args(cls, parser):
+        """Add listed args to command line parser."""
+        default_obj = cls()
+        num_ex_group = parser.add_mutually_exclusive_group(required=False)
+        num_ex_group.add_argument(
+            "--num-examples",
+            "-n",
+            type=int,
+            default=default_obj.num_examples,
+            help="The number of examples to process, -1 for entire dataset.",
+        )
+        num_ex_group.add_argument(
+            "--num-successful-examples",
+            type=int,
+            default=default_obj.num_successful_examples,
+            help="The number of successful adversarial examples we want.",
+        )
+        parser.add_argument(
+            "--num-examples-offset",
+            "-o",
+            type=int,
+            required=False,
+            default=default_obj.num_examples_offset,
+            help="The offset to start at in the dataset.",
+        )
+        parser.add_argument(
+            "--query-budget",
+            "-q",
+            type=int,
+            default=default_obj.query_budget,
+            help="The maximum number of model queries allowed per example attacked. Setting this overwrites the query budget set in `GoalFunction` object.",
+        )
+        parser.add_argument(
+            "--shuffle",
+            action="store_true",
+            default=default_obj.shuffle,
+            help="If `True`, shuffle the samples before we attack the dataset. Default is False.",
+        )
+        parser.add_argument(
+            "--attack-n",
+            action="store_true",
+            default=default_obj.attack_n,
+            help="Whether to run attack until `n` examples have been attacked (not skipped).",
+        )
+        parser.add_argument(
+            "--checkpoint-dir",
+            required=False,
+            type=str,
+            default=default_obj.checkpoint_dir,
+            help="The directory to save checkpoint files.",
+        )
+        parser.add_argument(
+            "--checkpoint-interval",
+            required=False,
+            type=int,
+            default=default_obj.checkpoint_interval,
+            help="If set, checkpoint will be saved after attacking every N examples. If not set, no checkpoints will be saved.",
+        )
+        parser.add_argument(
+            "--random-seed",
+            default=default_obj.random_seed,
+            type=int,
+            help="Random seed for reproducibility.",
+        )
+        parser.add_argument(
+            "--parallel",
+            action="store_true",
+            default=default_obj.parallel,
+            help="Run attack using multiple GPUs.",
+        )
+        parser.add_argument(
+            "--num-workers-per-device",
+            default=default_obj.num_workers_per_device,
+            type=int,
+            help="Number of worker processes to run per device.",
+        )
+        parser.add_argument(
+            "--log-to-txt",
+            nargs="?",
+            default=default_obj.log_to_txt,
+            const="",
+            type=str,
+            help="Path to which to save attack logs as a text file. Set this argument if you want to save text logs. "
+            "If the last part of the path ends with `.txt` extension, the path is assumed to path for output file.",
+        )
+        parser.add_argument(
+            "--log-to-csv",
+            nargs="?",
+            default=default_obj.log_to_csv,
+            const="",
+            type=str,
+            help="Path to which to save attack logs as a CSV file. Set this argument if you want to save CSV logs. "
+            "If the last part of the path ends with `.csv` extension, the path is assumed to path for output file.",
+        )
+        parser.add_argument(
+            "--log-summary-to-json",
+            nargs="?",
+            default=default_obj.log_summary_to_json,
+            const="",
+            type=str,
+            help="Path to which to save attack summary as a JSON file. Set this argument if you want to save attack results summary in a JSON. "
+            "If the last part of the path ends with `.json` extension, the path is assumed to path for output file.",
+        )
+        parser.add_argument(
+            "--csv-coloring-style",
+            default=default_obj.csv_coloring_style,
+            type=str,
+            help='Method for choosing how to mark perturbed parts of the text in CSV logs. Options are "file" and "plain". '
+            '"file" wraps text with double brackets `[[ <text> ]]` while "plain" does not mark any text. Default is "file".',
+        )
+        parser.add_argument(
+            "--log-to-visdom",
+            nargs="?",
+            default=None,
+            const='{"env": "main", "port": 8097, "hostname": "localhost"}',
+            type=json.loads,
+            help="Set this argument if you want to log attacks to Visdom. The dictionary should have the following "
+            'three keys and their corresponding values: `"env", "port", "hostname"`. '
+            'Example for command line use: `--log-to-visdom {"env": "main", "port": 8097, "hostname": "localhost"}`.',
+        )
+        parser.add_argument(
+            "--log-to-wandb",
+            nargs="?",
+            default=None,
+            const='{"project": "textattack"}',
+            type=json.loads,
+            help="Set this argument if you want to log attacks to WandB. The dictionary should have the following "
+            'key and its corresponding value: `"project"`. '
+            'Example for command line use: `--log-to-wandb {"project": "textattack"}`.',
+        )
+        parser.add_argument(
+            "--disable-stdout",
+            action="store_true",
+            default=default_obj.disable_stdout,
+            help="Disable logging attack results to stdout",
+        )
+        parser.add_argument(
+            "--silent",
+            action="store_true",
+            default=default_obj.silent,
+            help="Disable all logging",
+        )
+        parser.add_argument(
+            "--enable-advance-metrics",
+            action="store_true",
+            default=default_obj.enable_advance_metrics,
+            help="Enable calculation and display of optional advance post-hoc metrics like perplexity, USE distance, etc.",
+        )
+
+        return parser
+
+    @classmethod
+    def create_loggers_from_args(cls, args):
+        """Creates AttackLogManager from an AttackArgs object."""
+        assert isinstance(
+            args, cls
+        ), f"Expect args to be of type `{type(cls)}`, but got type `{type(args)}`."
+
+        # Create logger
+        attack_log_manager = textattack.loggers.AttackLogManager(args.metrics)
+
+        # Get current time for file naming
+        timestamp = time.strftime("%Y-%m-%d-%H-%M")
+
+        # if '--log-to-txt' specified with arguments
+        if args.log_to_txt is not None:
+            if args.log_to_txt.lower().endswith(".txt"):
+                txt_file_path = args.log_to_txt
+            else:
+                txt_file_path = os.path.join(args.log_to_txt, f"{timestamp}-log.txt")
+
+            dir_path = os.path.dirname(txt_file_path)
+            dir_path = dir_path if dir_path else "."
+            if not os.path.exists(dir_path):
+                os.makedirs(os.path.dirname(txt_file_path))
+
+            color_method = "file"
+            attack_log_manager.add_output_file(txt_file_path, color_method)
+
+        # if '--log-to-csv' specified with arguments
+        if args.log_to_csv is not None:
+            if args.log_to_csv.lower().endswith(".csv"):
+                csv_file_path = args.log_to_csv
+            else:
+                csv_file_path = os.path.join(args.log_to_csv, f"{timestamp}-log.csv")
+
+            dir_path = os.path.dirname(csv_file_path)
+            dir_path = dir_path if dir_path else "."
+            if not os.path.exists(dir_path):
+                os.makedirs(dir_path)
+
+            color_method = (
+                None if args.csv_coloring_style == "plain" else args.csv_coloring_style
+            )
+            attack_log_manager.add_output_csv(csv_file_path, color_method)
+
+        # if '--log-summary-to-json' specified with arguments
+        if args.log_summary_to_json is not None:
+            if args.log_summary_to_json.lower().endswith(".json"):
+                summary_json_file_path = args.log_summary_to_json
+            else:
+                summary_json_file_path = os.path.join(
+                    args.log_summary_to_json, f"{timestamp}-attack_summary_log.json"
+                )
+
+            dir_path = os.path.dirname(summary_json_file_path)
+            dir_path = dir_path if dir_path else "."
+            if not os.path.exists(dir_path):
+                os.makedirs(os.path.dirname(summary_json_file_path))
+
+            attack_log_manager.add_output_summary_json(summary_json_file_path)
+
+        # Visdom
+        if args.log_to_visdom is not None:
+            attack_log_manager.enable_visdom(**args.log_to_visdom)
+
+        # Weights & Biases
+        if args.log_to_wandb is not None:
+            attack_log_manager.enable_wandb(**args.log_to_wandb)
+
+        # Stdout
+        if not args.disable_stdout and not sys.stdout.isatty():
+            attack_log_manager.disable_color()
+        elif not args.disable_stdout:
+            attack_log_manager.enable_stdout()
+
+        return attack_log_manager
+
+
+@dataclass
+class _CommandLineAttackArgs:
+    """Attack args for command line execution. This requires more arguments to
+    create ``Attack`` object as specified.
+
+    Args:
+        transformation (:obj:`str`, `optional`, defaults to :obj:`"word-swap-embedding"`):
+            Name of transformation to use.
+        constraints (:obj:`list[str]`, `optional`, defaults to :obj:`["repeat", "stopword"]`):
+            List of names of constraints to use.
+        goal_function (:obj:`str`, `optional`, defaults to :obj:`"untargeted-classification"`):
+            Name of goal function to use.
+        search_method (:obj:`str`, `optional`, defualts to :obj:`"greedy-word-wir"`):
+            Name of search method to use.
+        attack_recipe (:obj:`str`, `optional`, defaults to :obj:`None`):
+            Name of attack recipe to use.
+            .. note::
+                Setting this overrides any previous selection of transformation, constraints, goal function, and search method.
+        attack_from_file (:obj:`str`, `optional`, defaults to :obj:`None`):
+            Path of `.py` file from which to load attack from. Use `<path>^<variable_name>` to specifiy which variable to import from the file.
+            .. note::
+                If this is set, it overrides any previous selection of transformation, constraints, goal function, and search method
+        interactive (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If `True`, carry attack in interactive mode.
+        parallel (:obj:`bool`, `optional`, defaults to :obj:`False`):
+            If `True`, attack in parallel.
+        model_batch_size (:obj:`int`, `optional`, defaults to :obj:`32`):
+            The batch size for making queries to the victim model.
+        model_cache_size (:obj:`int`, `optional`, defaults to :obj:`2**18`):
+            The maximum number of items to keep in the model results cache at once.
+        constraint-cache-size (:obj:`int`, `optional`, defaults to :obj:`2**18`):
+            The maximum number of items to keep in the constraints cache at once.
+    """
+
+    transformation: str = "word-swap-embedding"
+    constraints: list = field(default_factory=lambda: ["repeat", "stopword"])
+    goal_function: str = "untargeted-classification"
+    search_method: str = "greedy-word-wir"
+    attack_recipe: str = None
+    attack_from_file: str = None
+    interactive: bool = False
+    parallel: bool = False
+    model_batch_size: int = 32
+    model_cache_size: int = 2**18
+    constraint_cache_size: int = 2**18
+
+    @classmethod
+    def _add_parser_args(cls, parser):
+        """Add listed args to command line parser."""
+        default_obj = cls()
+        transformation_names = set(BLACK_BOX_TRANSFORMATION_CLASS_NAMES.keys()) | set(
+            WHITE_BOX_TRANSFORMATION_CLASS_NAMES.keys()
+        )
+        parser.add_argument(
+            "--transformation",
+            type=str,
+            required=False,
+            default=default_obj.transformation,
+            help='The transformation to apply. Usage: "--transformation {transformation}:{arg_1}={value_1},{arg_3}={value_3}". Choices: '
+            + str(transformation_names),
+        )
+        parser.add_argument(
+            "--constraints",
+            type=str,
+            required=False,
+            nargs="*",
+            default=default_obj.constraints,
+            help='Constraints to add to the attack. Usage: "--constraints {constraint}:{arg_1}={value_1},{arg_3}={value_3}". Choices: '
+            + str(CONSTRAINT_CLASS_NAMES.keys()),
+        )
+        goal_function_choices = ", ".join(GOAL_FUNCTION_CLASS_NAMES.keys())
+        parser.add_argument(
+            "--goal-function",
+            "-g",
+            default=default_obj.goal_function,
+            help=f"The goal function to use. choices: {goal_function_choices}",
+        )
+        attack_group = parser.add_mutually_exclusive_group(required=False)
+        search_choices = ", ".join(SEARCH_METHOD_CLASS_NAMES.keys())
+        attack_group.add_argument(
+            "--search-method",
+            "--search",
+            "-s",
+            type=str,
+            required=False,
+            default=default_obj.search_method,
+            help=f"The search method to use. choices: {search_choices}",
+        )
+        attack_group.add_argument(
+            "--attack-recipe",
+            "--recipe",
+            "-r",
+            type=str,
+            required=False,
+            default=default_obj.attack_recipe,
+            help="full attack recipe (overrides provided goal function, transformation & constraints)",
+            choices=ATTACK_RECIPE_NAMES.keys(),
+        )
+        attack_group.add_argument(
+            "--attack-from-file",
+            type=str,
+            required=False,
+            default=default_obj.attack_from_file,
+            help="Path of `.py` file from which to load attack from. Use `<path>^<variable_name>` to specifiy which variable to import from the file.",
+        )
+        parser.add_argument(
+            "--interactive",
+            action="store_true",
+            default=default_obj.interactive,
+            help="Whether to run attacks interactively.",
+        )
+        parser.add_argument(
+            "--model-batch-size",
+            type=int,
+            default=default_obj.model_batch_size,
+            help="The batch size for making calls to the model.",
+        )
+        parser.add_argument(
+            "--model-cache-size",
+            type=int,
+            default=default_obj.model_cache_size,
+            help="The maximum number of items to keep in the model results cache at once.",
+        )
+        parser.add_argument(
+            "--constraint-cache-size",
+            type=int,
+            default=default_obj.constraint_cache_size,
+            help="The maximum number of items to keep in the constraints cache at once.",
+        )
+
+        return parser
+
+    @classmethod
+    def _create_transformation_from_args(cls, args, model_wrapper):
+        """Create `Transformation` based on provided `args` and
+        `model_wrapper`."""
+
+        transformation_name = args.transformation
+        if ARGS_SPLIT_TOKEN in transformation_name:
+            transformation_name, params = transformation_name.split(ARGS_SPLIT_TOKEN)
+
+            if transformation_name in WHITE_BOX_TRANSFORMATION_CLASS_NAMES:
+                transformation = eval(
+                    f"{WHITE_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}(model_wrapper.model, {params})"
+                )
+            elif transformation_name in BLACK_BOX_TRANSFORMATION_CLASS_NAMES:
+                transformation = eval(
+                    f"{BLACK_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}({params})"
+                )
+            else:
+                raise ValueError(
+                    f"Error: unsupported transformation {transformation_name}"
+                )
+        else:
+            if transformation_name in WHITE_BOX_TRANSFORMATION_CLASS_NAMES:
+                transformation = eval(
+                    f"{WHITE_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}(model_wrapper.model)"
+                )
+            elif transformation_name in BLACK_BOX_TRANSFORMATION_CLASS_NAMES:
+                transformation = eval(
+                    f"{BLACK_BOX_TRANSFORMATION_CLASS_NAMES[transformation_name]}()"
+                )
+            else:
+                raise ValueError(
+                    f"Error: unsupported transformation {transformation_name}"
+                )
+        return transformation
+
+    @classmethod
+    def _create_goal_function_from_args(cls, args, model_wrapper):
+        """Create `GoalFunction` based on provided `args` and
+        `model_wrapper`."""
+
+        goal_function = args.goal_function
+        if ARGS_SPLIT_TOKEN in goal_function:
+            goal_function_name, params = goal_function.split(ARGS_SPLIT_TOKEN)
+            if goal_function_name not in GOAL_FUNCTION_CLASS_NAMES:
+                raise ValueError(
+                    f"Error: unsupported goal_function {goal_function_name}"
+                )
+            goal_function = eval(
+                f"{GOAL_FUNCTION_CLASS_NAMES[goal_function_name]}(model_wrapper, {params})"
+            )
+        elif goal_function in GOAL_FUNCTION_CLASS_NAMES:
+            goal_function = eval(
+                f"{GOAL_FUNCTION_CLASS_NAMES[goal_function]}(model_wrapper)"
+            )
+        else:
+            raise ValueError(f"Error: unsupported goal_function {goal_function}")
+        if args.query_budget:
+            goal_function.query_budget = args.query_budget
+        goal_function.model_cache_size = args.model_cache_size
+        goal_function.batch_size = args.model_batch_size
+        return goal_function
+
+    @classmethod
+    def _create_constraints_from_args(cls, args):
+        """Create list of `Constraints` based on provided `args`."""
+
+        if not args.constraints:
+            return []
+
+        _constraints = []
+        for constraint in args.constraints:
+            if ARGS_SPLIT_TOKEN in constraint:
+                constraint_name, params = constraint.split(ARGS_SPLIT_TOKEN)
+                if constraint_name not in CONSTRAINT_CLASS_NAMES:
+                    raise ValueError(f"Error: unsupported constraint {constraint_name}")
+                _constraints.append(
+                    eval(f"{CONSTRAINT_CLASS_NAMES[constraint_name]}({params})")
+                )
+            elif constraint in CONSTRAINT_CLASS_NAMES:
+                _constraints.append(eval(f"{CONSTRAINT_CLASS_NAMES[constraint]}()"))
+            else:
+                raise ValueError(f"Error: unsupported constraint {constraint}")
+
+        return _constraints
+
+    @classmethod
+    def _create_attack_from_args(cls, args, model_wrapper):
+        """Given ``CommandLineArgs`` and ``ModelWrapper``, return specified
+        ``Attack`` object."""
+
+        assert isinstance(
+            args, cls
+        ), f"Expect args to be of type `{type(cls)}`, but got type `{type(args)}`."
+
+        if args.attack_recipe:
+            if ARGS_SPLIT_TOKEN in args.attack_recipe:
+                recipe_name, params = args.attack_recipe.split(ARGS_SPLIT_TOKEN)
+                if recipe_name not in ATTACK_RECIPE_NAMES:
+                    raise ValueError(f"Error: unsupported recipe {recipe_name}")
+                recipe = eval(
+                    f"{ATTACK_RECIPE_NAMES[recipe_name]}.build(model_wrapper, {params})"
+                )
+            elif args.attack_recipe in ATTACK_RECIPE_NAMES:
+                recipe = eval(
+                    f"{ATTACK_RECIPE_NAMES[args.attack_recipe]}.build(model_wrapper)"
+                )
+            else:
+                raise ValueError(f"Invalid recipe {args.attack_recipe}")
+            if args.query_budget:
+                recipe.goal_function.query_budget = args.query_budget
+            recipe.goal_function.model_cache_size = args.model_cache_size
+            recipe.constraint_cache_size = args.constraint_cache_size
+            return recipe
+        elif args.attack_from_file:
+            if ARGS_SPLIT_TOKEN in args.attack_from_file:
+                attack_file, attack_name = args.attack_from_file.split(ARGS_SPLIT_TOKEN)
+            else:
+                attack_file, attack_name = args.attack_from_file, "attack"
+            attack_module = load_module_from_file(attack_file)
+            if not hasattr(attack_module, attack_name):
+                raise ValueError(
+                    f"Loaded `{attack_file}` but could not find `{attack_name}`."
+                )
+            attack_func = getattr(attack_module, attack_name)
+            return attack_func(model_wrapper)
+        else:
+            goal_function = cls._create_goal_function_from_args(args, model_wrapper)
+            transformation = cls._create_transformation_from_args(args, model_wrapper)
+            constraints = cls._create_constraints_from_args(args)
+            if ARGS_SPLIT_TOKEN in args.search_method:
+                search_name, params = args.search_method.split(ARGS_SPLIT_TOKEN)
+                if search_name not in SEARCH_METHOD_CLASS_NAMES:
+                    raise ValueError(f"Error: unsupported search {search_name}")
+                search_method = eval(
+                    f"{SEARCH_METHOD_CLASS_NAMES[search_name]}({params})"
+                )
+            elif args.search_method in SEARCH_METHOD_CLASS_NAMES:
+                search_method = eval(
+                    f"{SEARCH_METHOD_CLASS_NAMES[args.search_method]}()"
+                )
+            else:
+                raise ValueError(f"Error: unsupported attack {args.search_method}")
+
+        return Attack(
+            goal_function,
+            constraints,
+            transformation,
+            search_method,
+            constraint_cache_size=args.constraint_cache_size,
+        )
+
+
+# This neat trick allows use to reorder the arguments to avoid TypeErrors commonly found when inheriting dataclass.
+# https://stackoverflow.com/questions/51575931/class-inheritance-in-python-3-7-dataclasses
+@dataclass
+class CommandLineAttackArgs(AttackArgs, _CommandLineAttackArgs, DatasetArgs, ModelArgs):
+    @classmethod
+    def _add_parser_args(cls, parser):
+        """Add listed args to command line parser."""
+        parser = ModelArgs._add_parser_args(parser)
+        parser = DatasetArgs._add_parser_args(parser)
+        parser = _CommandLineAttackArgs._add_parser_args(parser)
+        parser = AttackArgs._add_parser_args(parser)
+        return parser

textattack/attack_recipes/__init__.py

@@ -0,0 +1,43 @@
+""".. _attack_recipes:
+
+Attack Recipes Package:
+========================
+
+We provide a number of pre-built attack recipes, which correspond to attacks from the literature. To run an attack recipe from the command line, run::
+
+    textattack attack --recipe [recipe_name]
+
+To initialize an attack in Python script, use::
+
+    <recipe name>.build(model_wrapper)
+
+For example, ``attack = InputReductionFeng2018.build(model)`` creates `attack`, an object of type ``Attack`` with the goal function, transformation, constraints, and search method specified in that paper. This object can then be used just like any other attack; for example, by calling ``attack.attack_dataset``.
+
+TextAttack supports the following attack recipes (each recipe's documentation contains a link to the corresponding paper):
+
+.. contents:: :local:
+"""
+
+from .attack_recipe import AttackRecipe
+
+from .a2t_yoo_2021 import A2TYoo2021
+from .bae_garg_2019 import BAEGarg2019
+from .bert_attack_li_2020 import BERTAttackLi2020
+from .genetic_algorithm_alzantot_2018 import GeneticAlgorithmAlzantot2018
+from .faster_genetic_algorithm_jia_2019 import FasterGeneticAlgorithmJia2019
+from .deepwordbug_gao_2018 import DeepWordBugGao2018
+from .hotflip_ebrahimi_2017 import HotFlipEbrahimi2017
+from .input_reduction_feng_2018 import InputReductionFeng2018
+from .kuleshov_2017 import Kuleshov2017
+from .morpheus_tan_2020 import MorpheusTan2020
+from .seq2sick_cheng_2018_blackbox import Seq2SickCheng2018BlackBox
+from .textbugger_li_2018 import TextBuggerLi2018
+from .textfooler_jin_2019 import TextFoolerJin2019
+from .pwws_ren_2019 import PWWSRen2019
+from .iga_wang_2019 import IGAWang2019
+from .pruthi_2019 import Pruthi2019
+from .pso_zang_2020 import PSOZang2020
+from .checklist_ribeiro_2020 import CheckList2020
+from .clare_li_2020 import CLARE2020
+from .french_recipe import FrenchRecipe
+from .spanish_recipe import SpanishRecipe

textattack/attack_recipes/a2t_yoo_2021.py

@@ -0,0 +1,74 @@
+"""
+A2T (A2T: Attack for Adversarial Training Recipe)
+==================================================
+
+"""
+
+from textattack import Attack
+from textattack.constraints.grammaticality import PartOfSpeech
+from textattack.constraints.pre_transformation import (
+    InputColumnModification,
+    MaxModificationRate,
+    RepeatModification,
+    StopwordModification,
+)
+from textattack.constraints.semantics import WordEmbeddingDistance
+from textattack.constraints.semantics.sentence_encoders import BERT
+from textattack.goal_functions import UntargetedClassification
+from textattack.search_methods import GreedyWordSwapWIR
+from textattack.transformations import WordSwapEmbedding, WordSwapMaskedLM
+
+from .attack_recipe import AttackRecipe
+
+
+class A2TYoo2021(AttackRecipe):
+    """Towards Improving Adversarial Training of NLP Models.
+
+    (Yoo et al., 2021)
+
+    https://arxiv.org/abs/2109.00544
+    """
+
+    @staticmethod
+    def build(model_wrapper, mlm=False):
+        """Build attack recipe.
+
+        Args:
+            model_wrapper (:class:`~textattack.models.wrappers.ModelWrapper`):
+                Model wrapper containing both the model and the tokenizer.
+            mlm (:obj:`bool`, `optional`, defaults to :obj:`False`):
+                If :obj:`True`, load `A2T-MLM` attack. Otherwise, load regular `A2T` attack.
+
+        Returns:
+            :class:`~textattack.Attack`: A2T attack.
+        """
+        constraints = [RepeatModification(), StopwordModification()]
+        input_column_modification = InputColumnModification(
+            ["premise", "hypothesis"], {"premise"}
+        )
+        constraints.append(input_column_modification)
+        constraints.append(PartOfSpeech(allow_verb_noun_swap=False))
+        constraints.append(MaxModificationRate(max_rate=0.1, min_threshold=4))
+        sent_encoder = BERT(
+            model_name="stsb-distilbert-base", threshold=0.9, metric="cosine"
+        )
+        constraints.append(sent_encoder)
+
+        if mlm:
+            transformation = transformation = WordSwapMaskedLM(
+                method="bae", max_candidates=20, min_confidence=0.0, batch_size=16
+            )
+        else:
+            transformation = WordSwapEmbedding(max_candidates=20)
+            constraints.append(WordEmbeddingDistance(min_cos_sim=0.8))
+
+        #
+        # Goal is untargeted classification
+        #
+        goal_function = UntargetedClassification(model_wrapper, model_batch_size=32)
+        #
+        # Greedily swap words with "Word Importance Ranking".
+        #
+        search_method = GreedyWordSwapWIR(wir_method="gradient")
+
+        return Attack(goal_function, constraints, transformation, search_method)

textattack/attack_recipes/attack_recipe.py

@@ -0,0 +1,30 @@
+"""
+Attack Recipe Class
+========================
+
+"""
+
+from abc import ABC, abstractmethod
+
+from textattack import Attack
+
+
+class AttackRecipe(Attack, ABC):
+    """A recipe for building an NLP adversarial attack from the literature."""
+
+    @staticmethod
+    @abstractmethod
+    def build(model_wrapper, **kwargs):
+        """Creates pre-built :class:`~textattack.Attack` that correspond to
+        attacks from the literature.
+
+        Args:
+            model_wrapper (:class:`~textattack.models.wrappers.ModelWrapper`):
+                :class:`~textattack.models.wrappers.ModelWrapper` that contains the victim model and tokenizer.
+                This is passed to :class:`~textattack.goal_functions.GoalFunction` when constructing the attack.
+            kwargs:
+                Additional keyword arguments.
+        Returns:
+            :class:`~textattack.Attack`
+        """
+        raise NotImplementedError()

textattack/attack_recipes/bae_garg_2019.py

@@ -0,0 +1,123 @@
+"""
+BAE (BAE: BERT-Based Adversarial Examples)
+============================================
+
+"""
+from textattack.constraints.grammaticality import PartOfSpeech
+from textattack.constraints.pre_transformation import (
+    RepeatModification,
+    StopwordModification,
+)
+from textattack.constraints.semantics.sentence_encoders import UniversalSentenceEncoder
+from textattack.goal_functions import UntargetedClassification
+from textattack.search_methods import GreedyWordSwapWIR
+from textattack.transformations import WordSwapMaskedLM
+
+from .attack_recipe import AttackRecipe
+
+
+class BAEGarg2019(AttackRecipe):
+    """Siddhant Garg and Goutham Ramakrishnan, 2019.
+
+    BAE: BERT-based Adversarial Examples for Text Classification.
+
+    https://arxiv.org/pdf/2004.01970
+
+    This is "attack mode" 1 from the paper, BAE-R, word replacement.
+
+    We present 4 attack modes for BAE based on the
+        R and I operations, where for each token t in S:
+        • BAE-R: Replace token t (See Algorithm 1)
+        • BAE-I: Insert a token to the left or right of t
+        • BAE-R/I: Either replace token t or insert a
+        token to the left or right of t
+        • BAE-R+I: First replace token t, then insert a
+        token to the left or right of t
+    """
+
+    @staticmethod
+    def build(model_wrapper):
+        # "In this paper, we present a simple yet novel technique: BAE (BERT-based
+        # Adversarial Examples), which uses a language model (LM) for token
+        # replacement to best fit the overall context. We perturb an input sentence
+        # by either replacing a token or inserting a new token in the sentence, by
+        # means of masking a part of the input and using a LM to fill in the mask."
+        #
+        # We only consider the top K=50 synonyms from the MLM predictions.
+        #
+        # [from email correspondance with the author]
+        # "When choosing the top-K candidates from the BERT masked LM, we filter out
+        # the sub-words and only retain the whole words (by checking if they are
+        # present in the GloVE vocabulary)"
+        #
+        transformation = WordSwapMaskedLM(
+            method="bae", max_candidates=50, min_confidence=0.0
+        )
+        #
+        # Don't modify the same word twice or stopwords.
+        #
+        constraints = [RepeatModification(), StopwordModification()]
+
+        # For the R operations we add an additional check for
+        # grammatical correctness of the generated adversarial example by filtering
+        # out predicted tokens that do not form the same part of speech (POS) as the
+        # original token t_i in the sentence.
+        constraints.append(PartOfSpeech(allow_verb_noun_swap=True))
+
+        # "To ensure semantic similarity on introducing perturbations in the input
+        # text, we filter the set of top-K masked tokens (K is a pre-defined
+        # constant) predicted by BERT-MLM using a Universal Sentence Encoder (USE)
+        # (Cer et al., 2018)-based sentence similarity scorer."
+        #
+        # "[We] set a threshold of 0.8 for the cosine similarity between USE-based
+        # embeddings of the adversarial and input text."
+        #
+        # [from email correspondence with the author]
+        # "For a fair comparison of the benefits of using a BERT-MLM in our paper,
+        # we retained the majority of TextFooler's specifications. Thus we:
+        # 1. Use the USE for comparison within a window of size 15 around the word
+        # being replaced/inserted.
+        # 2. Set the similarity score threshold to 0.1 for inputs shorter than the
+        # window size (this translates roughly to almost always accepting the new text).
+        # 3. Perform the USE similarity thresholding of 0.8 with respect to the text
+        # just before the replacement/insertion and not the original text (For
+        # example: at the 3rd R/I operation, we compute the USE score on a window
+        # of size 15 of the text obtained after the first 2 R/I operations and not
+        # the original text).
+        # ...
+        # To address point (3) from above, compare the USE with the original text
+        # at each iteration instead of the current one (While doing this change
+        # for the R-operation is trivial, doing it for the I-operation with the
+        # window based USE comparison might be more involved)."
+        #
+        # Finally, since the BAE code is based on the TextFooler code, we need to
+        # adjust the threshold to account for the missing / pi in the cosine
+        # similarity comparison. So the final threshold is 1 - (1 - 0.8) / pi
+        # = 1 - (0.2 / pi) = 0.936338023.
+        use_constraint = UniversalSentenceEncoder(
+            threshold=0.936338023,
+            metric="cosine",
+            compare_against_original=True,
+            window_size=15,
+            skip_text_shorter_than_window=True,
+        )
+        constraints.append(use_constraint)
+        #
+        # Goal is untargeted classification.
+        #
+        goal_function = UntargetedClassification(model_wrapper)
+        #
+        # "We estimate the token importance Ii of each token
+        # t_i ∈ S = [t1, . . . , tn], by deleting ti from S and computing the
+        # decrease in probability of predicting the correct label y, similar
+        # to (Jin et al., 2019).
+        #
+        # • "If there are multiple tokens can cause C to misclassify S when they
+        # replace the mask, we choose the token which makes Sadv most similar to
+        # the original S based on the USE score."
+        # • "If no token causes misclassification, we choose the perturbation that
+        # decreases the prediction probability P(C(Sadv)=y) the most."
+        #
+        search_method = GreedyWordSwapWIR(wir_method="delete")
+
+        return BAEGarg2019(goal_function, constraints, transformation, search_method)