Spaces:
Running
Running
| from pathlib import Path | |
| import io | |
| import json | |
| import math | |
| import statistics | |
| import sys | |
| import time | |
| from datasets import concatenate_datasets, Dataset | |
| from datasets import load_dataset | |
| from huggingface_hub import hf_hub_url | |
| import pandas as pd | |
| import numpy as np | |
| from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer | |
| from evaluate import load | |
| # 1. record each file name included | |
| # 1.1 read different file formats depending on parameters (i.e., filetype) | |
| # 2. determine column types and report how many rows for each type (format check) | |
| # (in a well-formatted dataset, each column should only have one type) | |
| # 3. report on the null values | |
| # 4. for certain column types, report statistics | |
| # 4.1 uniqueness: if all rows are of a small number of <string> values, treat the column as 'categorical' < 10. | |
| # 4.2 strings: length ranges | |
| # 4.3 lists: length ranges | |
| # 4.3 int/float/double: their percentiles, min, max, mean | |
| CELL_TYPES_LENGTH = ["<class 'str'>", "<class 'list'>"] | |
| CELL_TYPES_NUMERIC = ["<class 'int'>", "<class 'float'>"] | |
| PERCENTILES = [1, 5, 10, 25, 50, 100, 250, 500, 750, 900, 950, 975, 990, 995, 999] | |
| def read_data(all_files, filetype): | |
| df = None | |
| func_name = "" | |
| if filetype in ["parquet", "csv", "json"]: | |
| if filetype == "parquet": | |
| func_name = pd.read_parquet | |
| elif filetype == "csv": | |
| func_name = pd.read_csv | |
| elif filetype == "json": | |
| func_name = pd.read_json | |
| df = pd.concat(func_name(f) for f in all_files) | |
| elif filetype == "arrow": | |
| ds = concatenate_datasets([Dataset.from_file(str(fname)) for fname in all_files]) | |
| df = pd.DataFrame(data=ds) | |
| elif filetype == "jsonl": | |
| func_name = pd.read_json | |
| all_lines = [] | |
| for fname in all_files: | |
| with open(fname, "r") as f: | |
| all_lines.extend(f.readlines()) | |
| df = pd.concat([pd.DataFrame.from_dict([json.loads(line)]) for line in all_lines]) | |
| return df | |
| def compute_cell_length_ranges(cell_lengths, cell_unique_string_values): | |
| cell_length_ranges = {} | |
| cell_length_ranges = {} | |
| string_categorical = {} | |
| # this is probably a 'categorical' (i.e., 'classes' in HuggingFace) value | |
| # with few unique items (need to check that while reading the cell), | |
| # so no need to treat it as a normal string | |
| if len(cell_unique_string_values) > 0 and len(cell_unique_string_values) <= 10: | |
| string_categorical = str(len(cell_unique_string_values)) + " class(es)" | |
| elif cell_lengths: | |
| cell_lengths = sorted(cell_lengths) | |
| min_val = cell_lengths[0] | |
| max_val = cell_lengths[-1] | |
| distance = math.ceil((max_val - min_val) / 10.0) | |
| ranges = [] | |
| if min_val != max_val: | |
| for j in range(min_val, max_val, distance): | |
| ranges.append(j) | |
| for j in range(len(ranges)-1): | |
| cell_length_ranges[str(ranges[j]) + "-" + str(ranges[j+1])] = 0 | |
| ranges.append(max_val) | |
| j = 1 | |
| c = 0 | |
| for k in cell_lengths: | |
| if j == len(ranges): | |
| c += 1 | |
| elif k < ranges[j]: | |
| c += 1 | |
| else: | |
| cell_length_ranges[str(ranges[j-1]) + "-" + str(ranges[j])] = c | |
| j += 1 | |
| c = 1 | |
| cell_length_ranges[str(ranges[j-1]) + "-" + str(max_val)] = c | |
| else: | |
| ranges = [min_val] | |
| c = 0 | |
| for k in cell_lengths: | |
| c += 1 | |
| cell_length_ranges[str(min_val)] = c | |
| return cell_length_ranges, string_categorical | |
| def _compute_percentiles(values, percentiles=PERCENTILES): | |
| result = {} | |
| quantiles = statistics.quantiles(values, n=max(PERCENTILES)+1, method='inclusive') | |
| for p in percentiles: | |
| result[p/10] = quantiles[p-1] | |
| return result | |
| def compute_cell_value_statistics(cell_values): | |
| stats = {} | |
| if cell_values: | |
| cell_values = sorted(cell_values) | |
| stats["min"] = cell_values[0] | |
| stats["max"] = cell_values[-1] | |
| stats["mean"] = statistics.mean(cell_values) | |
| stats["stdev"] = statistics.stdev(cell_values) | |
| stats["variance"] = statistics.variance(cell_values) | |
| stats["percentiles"] = _compute_percentiles(cell_values) | |
| return stats | |
| def check_null(cell, cell_type): | |
| if cell_type == "<class 'float'>": | |
| if math.isnan(cell): | |
| return True | |
| elif cell is None: | |
| return True | |
| return False | |
| def compute_property(data_path, glob, filetype): | |
| output = {} | |
| data_dir = Path(data_path) | |
| filenames = [] | |
| all_files = list(data_dir.glob(glob)) | |
| for f in all_files: | |
| print(str(f)) | |
| base_fname = str(f)[len(str(data_path)):] | |
| if not data_path.endswith("/"): | |
| base_fname = base_fname[1:] | |
| filenames.append(base_fname) | |
| output["filenames"] = filenames | |
| df = read_data(all_files, filetype) | |
| column_info = {} | |
| for col_name in df.columns: | |
| if col_name not in column_info: | |
| column_info[col_name] = {} | |
| cell_types = {} | |
| cell_lengths = {} | |
| cell_unique_string_values = {} | |
| cell_values = {} | |
| null_count = 0 | |
| col_values = df[col_name].to_list() | |
| for cell in col_values: | |
| # for index, row in df.iterrows(): | |
| # cell = row[col_name] | |
| cell_type = str(type(cell)) | |
| cell_type = str(type(cell)) | |
| # print(cell, cell_type) | |
| if check_null(cell, cell_type): | |
| null_count += 1 | |
| continue | |
| if cell_type not in cell_types: | |
| cell_types[cell_type] = 1 | |
| else: | |
| cell_types[cell_type] += 1 | |
| if cell_type in CELL_TYPES_LENGTH: | |
| cell_length = len(cell) | |
| if cell_type not in cell_lengths: | |
| cell_lengths[cell_type] = [] | |
| cell_lengths[cell_type].append(cell_length) | |
| if cell_type == "<class 'str'>" and cell not in cell_unique_string_values: | |
| cell_unique_string_values[cell] = True | |
| elif cell_type in CELL_TYPES_NUMERIC: | |
| if cell_type not in cell_values: | |
| cell_values[cell_type] = [] | |
| cell_values[cell_type].append(cell) | |
| else: | |
| print(cell_type) | |
| clrs = {} | |
| ccs = {} | |
| for cell_type in CELL_TYPES_LENGTH: | |
| if cell_type in cell_lengths: | |
| clr, cc = compute_cell_length_ranges(cell_lengths[cell_type], cell_unique_string_values) | |
| clrs[cell_type] = clr | |
| ccs[cell_type] = cc | |
| css = {} | |
| for cell_type in CELL_TYPES_NUMERIC: | |
| if cell_type in cell_values: | |
| cell_stats = compute_cell_value_statistics(cell_values[cell_type]) | |
| css[cell_type] = cell_stats | |
| column_info[col_name]["cell_types"] = cell_types | |
| column_info[col_name]["cell_length_ranges"] = clrs | |
| column_info[col_name]["cell_categories"] = ccs | |
| column_info[col_name]["cell_stats"] = css | |
| column_info[col_name]["cell_missing"] = null_count | |
| output["column_info"] = column_info | |
| output["number_of_items"] = len(df) | |
| output["timestamp"] = time.time() | |
| return output | |
| def preprocess_function(examples): | |
| return tokenizer(examples["sentence1"], examples["sentence2"], truncation=True) | |
| def compute_metrics(eval_pred): | |
| predictions, labels = eval_pred | |
| predictions = np.argmax(predictions, axis=1) | |
| return metric.compute(predictions=predictions, references=labels) | |
| def compute_model_card_evaluation_results(tokenizer, model_checkpoint, raw_datasets, metric): | |
| tokenized_datasets = raw_datasets.map(preprocess_function, batched=True) | |
| model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=2) | |
| batch_size = 16 | |
| args = TrainingArguments( | |
| "test-glue", | |
| evaluation_strategy = "epoch", | |
| learning_rate=5e-5, | |
| seed=42, | |
| lr_scheduler_type="linear", | |
| per_device_train_batch_size=batch_size, | |
| per_device_eval_batch_size=batch_size, | |
| num_train_epochs=3, | |
| weight_decay=0.01, | |
| load_best_model_at_end=False, | |
| metric_for_best_model="accuracy", | |
| report_to="none" | |
| ) | |
| trainer = Trainer( | |
| model, | |
| args, | |
| train_dataset=tokenized_datasets["train"], | |
| eval_dataset=tokenized_datasets["validation"], | |
| tokenizer=tokenizer, | |
| compute_metrics=compute_metrics | |
| ) | |
| result = trainer.evaluate() | |
| return result | |
| if __name__ == "__main__": | |
| in_container = True | |
| if len(sys.argv) > 1: | |
| model_checkpoint = sys.argv[1] | |
| dataset_name = sys.argv[2] | |
| metric = sys.argv[3] | |
| in_container = False | |
| else: | |
| model_checkpoint = "sgugger/glue-mrpc" | |
| dataset_name = "nyu-mll/glue" | |
| metric = ["glue", "mrpc"] | |
| in_container = False | |
| print(model_checkpoint, dataset_name, metric) | |
| model_checkpoint = model_checkpoint | |
| raw_datasets = load_dataset(dataset_name, "mrpc") | |
| metric = load("glue", "mrpc") | |
| tokenizer = AutoTokenizer.from_pretrained(model_checkpoint) | |
| output = compute_model_card_evaluation_results(tokenizer, model_checkpoint, raw_datasets, metric) | |
| print(json.dumps(output)) | |
| if in_container: | |
| with open("/tmp/outputs/computation_result.json", "w") as f: | |
| json.dump(output, f, indent=4, sort_keys=True) | |
| else: | |
| print(json.dumps(output, indent=4, sort_keys=True)) |