git lfs install
git clone https://huggingface.co/bert-base-chinese

python
>>> from huggingface_hub import snapshot_download
>>> snapshot_download(repo_id="facebook/opt-13b")

# 通过参数可以指定下载的文件，屏蔽不需要的文件即可
>>> snapshot_download(repo_id="facebook/opt-13b", ignore_patterns=["*.h5", "*.ot", "*.msgpack"], local_dir="/localdata_ssd/model", local_dir_use_symlinks=False)

from transformers import pipeline
classifier = pipeline("sentiment-analysis")
print(classifier("You like a pig"))

output: [{'label': 'NEGATIVE', 'score': 0.9961085915565491}]

from transformers import AutoConfig
my_config = AutoConfig.from_pretrained("distilbert-base-uncased", n_heads=12)
my_model = AutoModel.from_config(my_config)

from transformers import AutoModelForSequenceClassification
model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased")

from transformers import TrainingArguments
training_args = TrainingArguments(
    output_dir="path/to/save/folder/",
    learning_rate=2e-5,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=8,
    num_train_epochs=2,
)

from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")

train_dataset = dataset["train"]  # doctest: +SKIP
eval_dataset = dataset["eval"]  # doctest: +SKIP

from transformers import DefaultDataCollator
data_collator = DefaultDataCollator()

from transformers import Trainer

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset["train"],
    eval_dataset=dataset["test"],
    tokenizer=tokenizer,
    data_collator=data_collator,
)  # doctest: +SKIP

# start training
trainer.train()

batch_sentences = [
    "But what about second breakfast?",
    "Don't think he knows about second breakfast, Pip.",
    "What about elevensies?",
]
encoded_input = tokenizer(batch_sentences, padding=True)
print(encoded_input)
{'input_ids': [[101, 1252, 1184, 1164, 1248, 6462, 136, 102, 0, 0, 0, 0, 0, 0, 0], 
               [101, 1790, 112, 189, 1341, 1119, 3520, 1164, 1248, 6462, 117, 21902, 1643, 119, 102], 
               [101, 1327, 1164, 5450, 23434, 136, 102, 0, 0, 0, 0, 0, 0, 0, 0]], 
 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 
 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0], 
                    [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 
                    [1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]]}

# Download from huggingface.co and cache
model = BertModel.from_pretrained("bert-base-uncased")
model.save_pretrained('./local_model_directory/')

# Load from local
model = BertModel.from_pretrained('./local_model_directory/')

if(os.path.exists('./model/'+args.gpt2_model+'pytorch_model.bin')):
    model.GPT2ForSequenceClassification.from_pretrained('./model/'+args.gpt2_model)
else:
    model = GPT2ForSequenceClassification.from_pretrained(args.gpt2_model)
    num_labels = len(model.config.id2label)
    model = GPT2ForSequenceClassification.from_pretrained(args.gpt2_model, num_labels=num_labels)
    model.save_pretrained('./model/'+args.gpt2_model)

# load
from datasets import load_dataset
dataset = load_dataset("yelp_review_full")
dataset["train"][100]
{'label': 0,
 'text': 'My expectations for McDonalds are t rarely high....'}

# tokenizer
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=True)
tokenized_datasets = dataset.map(tokenize_function, batched=True)

from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset)
train_features = get_train_features(
    args.data_dir,
    args.gpt2_model,
    args.max_seq_length,
    args.do_lower_case,
    args.local_rank,
    args.train_batch_size,
    args.gradient_accumulation_steps,
    args.num_train_epochs,
    tokenizer,
    processor,
)
train_data = gen_tensor_dataset(train_features)
        train_dataloader = DataLoader(
            train_data,
            sampler=train_sampler,
            batch_size=args.train_batch_size,
        )
for e in range(args.num_train_epochs):
    ...
    for step, batch in enumerate(train_dataloader):
        ...

model, optimizer, scheduler = init_optimizer_and_amp(
    model,
    args.learning_rate,
    args.loss_scale,
    args.warmup_proportion,
    num_train_optimization_steps,
    args.fp16,
)
def init_optimizer_and_amp(model, learning_rate, loss_scale, warmup_proportion,
                           num_train_optimization_steps, use_fp16):
    ...
    optimizer, scheduler = None, None
    logger.info("using fp32")
    if num_train_optimization_steps is not None:
        optimizer = BertAdam(
            optimizer_grouped_parameters,
            lr=learning_rate,
            warmup=warmup_proportion,
            t_total=num_train_optimization_steps,
        )
    return model, optimizer, scheduler

model.train()
for e in range(args.num_train_epochs):
    ...
    for step, batch in enumerate(train_dataloader):
        batch = tuple(t.to(device) for t in batch)
        input_ids, input_mask, segment_ids, label_ids = batch
        # outputs = model(**batch)
        outputs = model(input_ids=input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=label_ids)
        loss = outputs.loss
        loss.backward()

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()
        progress_bar.update(1)

model.eval()
for i, (input_ids, input_mask, segment_ids, label_ids) in enumerate(eval_dataloader):
    input_ids = input_ids.to(device)
    input_mask = input_mask.to(device)
    segment_ids = segment_ids.to(device)
    label_ids = label_ids.to(device)
    with torch.no_grad():
        # label_ids = label_ids.to(torch.float)
        outputs = model(input_ids=input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=label_ids)
        logits = outputs.logits
    ...

torch.cuda.set_device(i)

# i from 0-N-1，每个进程中都需要：
torch.distributed.init_process_group(
    backend='nccl', world_size=N, init_method='...'
)
model = DistributedDataParallel(model, device_ids=[i], output_device=i)

# 实践
if args.local_rank == -1 or args.no_cuda:
    device = torch.device(
        "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
    n_gpu = torch.cuda.device_count()
else:
    torch.cuda.set_device(args.local_rank)
    device = torch.device("cuda", args.local_rank)
    n_gpu = 1
    # Initializes the distributed backend which will take care of
    # sychronizing nodes/GPUs.
    if not torch.distributed.is_initialized():
        torch.distributed.init_process_group(backend='nccl')

from accelerator import Accelerator

accelerator = Accelerator()

model, optimizer, train_dataloader, lr_scheduler = accelerator.prepare(
    model, optimizer, train_dataloader, lr_scheduler
)

validation_dataloader = accelerator.prepare(validation_dataloader)

for inputs, targets in validation_dataloader:
    predictions = model(inputs)
    # Gather all predictions and targets
    all_predictions, all_targets = accelerator.gather_for_metrics((predictions, targets))
    # Example of use with a *Datasets.Metric*
    metric.add_batch(all_predictions, all_targets)

# 单 GPU 执行
CUDA_VISIBLE_DEVICES="0" accelerate launch {script_name.py} --arg1 --arg2 ...

# 默认参数，使用所有的 GPU，不启动混合精度
accelerate launch --multi_gpu {script_name.py} {--arg1} {--arg2} ...

# 当然，也可以指定参数
accelerate launch --multi_gpu --mixed_precision=fp16 --num_processes=2 {script_name.py} {--arg1} {--arg2} ...

# 查看所有参数
accelerate launch -h

accelerate launch --config_file ~/.cache/huggingface/accelerate {script_name.py} {--arg1} {--arg2} ...

# 搜索到数据集 oscar，里面有很多 subset，比如：unshuffled_deduplicated_en, unshuffled_deduplicated_br

# 第一个参数是数据集的名称，第二个参数用于索引子集，一般还有 split 参数：train, test, validation 

from datasets import load_dataset
dataset = load_dataset(path="oscar", name="unshuffled_deduplicated_en")
# 420G，赶紧中断
dataset.save_to_disk("./glue/")

# 尝试下载 wikitext
dataset = load_dataset(path="wikitext", name="wikitext-103-v1", split="train")
# 2022-10-24 14:07:31，是可以的
print(dataset[:3])
{'text': ['', ' = Valkyria Chronicles III = \n', '']}

# 保存
dataset.save_to_disk("./")

from datasets import load_dataset
# 如果用了 split 
dataset = load_dataset("rotten_tomatoes", split="train")
print(dataset)
Dataset({
    features: ['text', 'label'],
    num_rows: 8530
})
# 如果不用 split，
dataset = load_dataset("rotten_tomatoes")
DatasetDict({
    train: Dataset({
        features: ['text', 'label'],
        num_rows: 8530
    })
    validation: Dataset({
        features: ['text', 'label'],
        num_rows: 1066
    })
    test: Dataset({
        features: ['text', 'label'],
        num_rows: 1066
    })
})

{
  "attention_probs_dropout_prob": 0.1,
  "hidden_act": "gelu",
  "hidden_dropout_prob": 0.1,
  "hidden_size": 768,
  "initializer_range": 0.02,
  "intermediate_size": 3072,
  "max_position_embeddings": 512,
  "num_attention_heads": 12,
  "num_hidden_layers": 12,
  "type_vocab_size": 2,
  "vocab_size": 30522
}

pip install transformers
ln -s ../BERT/glue/ ./glue

from datasets import load_dataset
test_dataset = load_dataset("json", data_files="test.json", split="train")
test_dataset.save_to_disk("test.hf")

row_datasets = load_dataset(data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir)
# tokenizer
tokenized_datasets = raw_datasets.map(
    tokenize_function,
    batched=True,
    num_proc=data_args.preprocessing_num_workers,
    remove_columns=column_names,
    load_from_cache_file=not data_args.overwrite_cache,
    desc="Running tokenizer on dataset",
)
with training_args.main_process_first(desc="grouping texts together"):
    lm_datasets = tokenized_datasets.map(
        group_texts,
        batched=True,
        num_proc=data_args.preprocessing_num_workers,
        load_from_cache_file=not data_args.overwrite_cache,
        desc=f"Grouping texts in chunks of {block_size}",
    )
# 计算 token 长度
for split, data in lm_datasets.items():
total_eval_tokens = 0        
for chunk in data['labels']:
    total_eval_tokens += len([x for x in chunk[1:] if x != padding_index])
logger.info(f'[{split}] Total eval tokens: {total_eval_tokens}')

# 得到 eval dataset 以及 train dataset
if training_args.do_train:
    if "train" not in tokenized_datasets:
        raise ValueError("--do_train requires a train dataset")
    train_dataset = lm_datasets["train"]
    if data_args.max_train_samples is not None:
        train_dataset = train_dataset.select(range(data_args.max_train_samples))

if training_args.do_eval:
    if "validation" not in tokenized_datasets:
        raise ValueError("--do_eval requires a validation dataset")
    eval_dataset = lm_datasets[data_args.eval_subset]
    if data_args.max_eval_samples is not None:
        eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))

# 这个时候 eval_dataset size (480, 3)，3 个维度分别是 input_ids, attention_mask, labels

MODEL=t5-small

CUDA_VISIBLE_DEVICES=0 python -u run_translation.py  \
  --model_name_or_path ${MODEL} \
  --dataset_name wmt16 --dataset_config_name ro-en \
  --per_device_eval_batch_size=4 \
  --output_dir checkpoints-translation/${MODEL} \
  --source_lang en --target_lang ro \
  --do_eval \
  --predict_with_generate \
  --source_prefix "translate English to Romanian: "

from transformers import BertTokenizer, BertModel
import torch

tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
model = BertModel.from_pretrained("bert-base-uncased")

inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
print(inputs)
{'input_ids': tensor([[  101,  7592,  1010,  2026,  3899,  2003, 10140,   102]]), 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1]])}

outputs = model(**inputs)
# output 输出包含 last_hidden_state, pooler_output，其他属性 hidden_states, past_key_values, attentions, cross_attentions 为 None

from torch import nn
from transformers import Trainer


class CustomTrainer(Trainer):
    def compute_loss(self, model, inputs, return_outputs=False):
        labels = inputs.get("labels")
        # forward pass
        outputs = model(**inputs)
        logits = outputs.get("logits")
        # compute custom loss (suppose one has 3 labels with different weights)
        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor([1.0, 2.0, 3.0]))
        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))
        return (loss, outputs) if return_outputs else loss

label_ids = label_ids.to(torch.float)
outputs = model(input_ids=input_ids, token_type_ids=segment_ids, attention_mask=input_mask, labels=label_ids)
logits = outputs.logits
loss = outputs.loss