feat(transformers/models): add models of longt5, longformer, etc. (mindspore-lab#1234)

Author: The-truthh

mindone/transformers/__init__.py

@@ -655,6 +655,16 @@
     LlavaOnevisionProcessor,
     LlavaOnevisionVideoProcessor,
 )
+from .models.longformer import (
+    LongformerForMaskedLM,
+    LongformerForMultipleChoice,
+    LongformerForQuestionAnswering,
+    LongformerForSequenceClassification,
+    LongformerForTokenClassification,
+    LongformerModel,
+    LongformerPreTrainedModel,
+)
+from .models.longt5 import LongT5EncoderModel, LongT5ForConditionalGeneration, LongT5Model, LongT5PreTrainedModel
 from .models.luke import (
     LukeForEntityClassification,
     LukeForEntityPairClassification,
@@ -1020,6 +1030,7 @@
     TapasModel,
     TapasPreTrainedModel,
 )
+from .models.timesformer import TimesformerForVideoClassification, TimesformerModel, TimesformerPreTrainedModel
 from .models.trocr import TrOCRForCausalLM, TrOCRPreTrainedModel
 from .models.tvp import TvpForVideoGrounding, TvpModel, TvpPreTrainedModel
 from .models.umt5 import (
@@ -1076,6 +1087,7 @@
 from .models.vitpose import VitPoseForPoseEstimation, VitPosePreTrainedModel
 from .models.vitpose_backbone import VitPoseBackbone, VitPoseBackbonePreTrainedModel
 from .models.vits import VitsModel, VitsPreTrainedModel
+from .models.vivit import VivitForVideoClassification, VivitModel, VivitPreTrainedModel
 from .models.wav2vec2 import (
     Wav2Vec2FeatureExtractor,
     Wav2Vec2ForAudioFrameClassification,

mindone/transformers/activations.py

@@ -225,3 +225,6 @@ def get_activation(activation_string):
         return ACT2FN[activation_string]
     else:
         raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACT2FN.keys())}")
+
+
+gelu = get_activation("gelu")

mindone/transformers/models/__init__.py

@@ -102,6 +102,8 @@
     llava_next,
     llava_next_video,
     llava_onevision,
+    longformer,
+    longt5,
     luke,
     m2m_100,
     mamba,
@@ -169,6 +171,7 @@
     switch_transformers,
     t5,
     tapas,
+    timesformer,
     trocr,
     tvp,
     umt5,
@@ -187,6 +190,7 @@
     vitpose,
     vitpose_backbone,
     vits,
+    vivit,
     wav2vec2,
     x_clip,
     xlm_roberta,

mindone/transformers/models/auto/configuration_auto.py

@@ -126,6 +126,8 @@
         ("llava_next", "LlavaNextConfig"),
         ("llava_next_video", "LlavaNextVideoConfig"),
         ("llava_onevision", "LlavaOnevisionConfig"),
+        ("longformer", "LongformerConfig"),
+        ("longt5", "LongT5Config"),
         ("luke", "LukeConfig"),
         ("mamba", "MambaConfig"),
         ("mamba2", "Mamba2Config"),
@@ -193,6 +195,7 @@
         ("swin2sr", "Swin2SRConfig"),
         ("t5", "T5Config"),
         ("tapas", "TapasConfig"),
+        ("timesformer", "TimesformerConfig"),
         ("trocr", "TrOCRConfig"),
         ("tvp", "TvpConfig"),
         ("umt5", "UMT5Config"),
@@ -209,6 +212,7 @@
         ("vitdet", "VitDetConfig"),
         ("vitpose", "VitPoseConfig"),
         ("vitpose_backbone", "VitPoseBackboneConfig"),
+        ("vivit", "VivitConfig"),
         ("wav2vec2", "Wav2Vec2Config"),
         ("mvp", "MvpConfig"),
         ("whisper", "WhisperConfig"),
@@ -330,6 +334,8 @@
         ("llava_next", "LLaVA-NeXT"),
         ("llava_next_video", "LLaVa-NeXT-Video"),
         ("llava_onevision", "LLaVA-Onevision"),
+        ("longformer", "Longformer"),
+        ("longt5", "LongT5"),
         ("mimi", "Mimi"),
         ("mistral", "Mistral"),
         ("mllama", "Mllama"),
@@ -401,6 +407,7 @@
         ("t5", "T5"),
         ("t5v1.1", "T5v1.1"),
         ("tapas", "TAPAS"),
+        ("timesformer", "TimeSformer"),
         ("trocr", "TrOCR"),
         ("tvp", "TVP"),
         ("umt5", "UMT5"),
@@ -417,6 +424,7 @@
         ("vitdet", "VitDet"),
         ("vitpose", "ViTPose"),
         ("vitpose_backbone", "ViTPoseBackbone"),
+        ("vivit", "ViViT"),
         ("wav2vec2", "Wav2Vec2"),
         ("whisper", "Whisper"),
         ("xclip", "X-CLIP"),

mindone/transformers/models/auto/modeling_auto.py

@@ -119,6 +119,8 @@
         ("levit", "LevitModel"),
         ("lilt", "LiltModel"),
         ("llama", "LlamaModel"),
+        ("longformer", "LongformerModel"),
+        ("longt5", "LongT5Model"),
         ("luke", "LukeModel"),
         ("m2m_100", "M2M100Model"),
         ("mamba", "MambaModel"),
@@ -177,6 +179,7 @@
         ("swin2sr", "Swin2SRModel"),
         ("t5", "T5Model"),
         ("tapas", "TapasModel"),
+        ("timesformer", "TimesformerModel"),
         ("tvp", "TvpModel"),
         ("umt5", "UMT5Model"),
         ("unispeech", "UniSpeechModel"),
@@ -187,6 +190,7 @@
         ("vit", "ViTModel"),
         ("vit_msn", "ViTMSNModel"),
         ("vitdet", "VitDetModel"),
+        ("vivit", "VivitModel"),
         ("wav2vec2", "Wav2Vec2Model"),
         ("whisper", "WhisperModel"),
         ("xclip", "XCLIPModel"),
@@ -234,7 +238,11 @@
         ("llava_next", "LlavaNextForConditionalGeneration"),
         ("llava_next_video", "LlavaNextVideoForConditionalGeneration"),
         ("llava_onevision", "LlavaOnevisionForConditionalGeneration"),
+        ("longformer", "LongformerForMaskedLM"),
         ("luke", "LukeForMaskedLM"),
+        ("mobilebert", "MobileBertForPreTraining"),
+        ("qwen2_audio", "Qwen2AudioForConditionalGeneration"),
+        ("roberta", "RobertaForMaskedLM"),
         ("megatron-bert", "MegatronBertForPreTraining"),
         ("mistral3", "Mistral3ForConditionalGeneration"),
         ("mllama", "MllamaForConditionalGeneration"),
@@ -287,7 +295,10 @@
         ("gpt2", "GPT2LMHeadModel"),
         ("ibert", "IBertForMaskedLM"),
         ("led", "LEDForConditionalGeneration"),
+        ("longformer", "LongformerForMaskedLM"),
+        ("longt5", "LongT5ForConditionalGeneration"),
         ("luke", "LukeForMaskedLM"),
+        ("camembert", "CamembertForMaskedLM"),
         ("roberta", "RobertaForMaskedLM"),
         ("mamba", "MambaForCausalLM"),
         ("mamba2", "Mamba2ForCausalLM"),
@@ -413,9 +424,11 @@
         ("segformer", "SegformerModel"),
         ("siglip_vision_model", "SiglipVisionModel"),
         ("swin2sr", "Swin2SRModel"),
+        ("timesformer", "TimesformerModel"),
         ("vit", "ViTModel"),
         ("vit_msn", "ViTMSNModel"),
         ("vitdet", "VitDetModel"),
+        ("vivit", "VivitModel"),
         ("yolos", "YolosModel"),
         ("zamba2", "Zamba2ForCausalLM"),
     ]
@@ -503,7 +516,12 @@
     ]
 )
 
-MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES = OrderedDict()
+MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES = OrderedDict(
+    [
+        ("timesformer", "TimesformerForVideoClassification"),
+        ("vivit", "VivitForVideoClassification"),
+    ]
+)
 
 MODEL_FOR_VISION_2_SEQ_MAPPING_NAMES = OrderedDict(
     [
@@ -584,6 +602,7 @@
         ("electra", "ElectraForMaskedLM"),
         ("funnel", "FunnelForMaskedLM"),
         ("ibert", "IBertForMaskedLM"),
+        ("longformer", "LongformerForMaskedLM"),
         ("luke", "LukeForMaskedLM"),
         ("mobilebert", "MobileBertForMaskedLM"),
         ("mpnet", "MPNetForMaskedLM"),
@@ -640,6 +659,7 @@
         ("blenderbot-small", "BlenderbotSmallForConditionalGeneration"),
         ("fsmt", "FSMTForConditionalGeneration"),
         ("led", "LEDForConditionalGeneration"),
+        ("longt5", "LongT5ForConditionalGeneration"),
         ("m2m_100", "M2M100ForConditionalGeneration"),
         ("mvp", "MvpForConditionalGeneration"),
         ("nllb-moe", "NllbMoeForConditionalGeneration"),
@@ -698,6 +718,7 @@
         ("canine", "CanineForSequenceClassification"),
         ("lilt", "LiltForSequenceClassification"),
         ("llama", "LlamaForSequenceClassification"),
+        ("longformer", "LongformerForSequenceClassification"),
         ("opt", "OPTForSequenceClassification"),
         ("persimmon", "PersimmonForSequenceClassification"),
         ("mbart", "MBartForSequenceClassification"),
@@ -752,6 +773,7 @@
         ("luke", "LukeForQuestionAnswering"),
         ("convbert", "ConvBertForQuestionAnswering"),
         ("llama", "LlamaForQuestionAnswering"),
+        ("longformer", "LongformerForQuestionAnswering"),
         ("mistral", "MistralForQuestionAnswering"),
         ("mobilebert", "MobileBertForQuestionAnswering"),
         ("mpnet", "MPNetForQuestionAnswering"),
@@ -812,6 +834,7 @@
         ("helium", "HeliumForTokenClassification"),
         ("ibert", "IBertForTokenClassification"),
         ("lilt", "LiltForTokenClassification"),
+        ("longformer", "LongformerForTokenClassification"),
         ("luke", "LukeForTokenClassification"),
         ("mistral", "MistralForTokenClassification"),
         ("mobilebert", "MobileBertForTokenClassification"),
@@ -852,6 +875,7 @@
         ("distilbert", "DistilBertForMultipleChoice"),
         ("funnel", "FunnelForMultipleChoice"),
         ("ibert", "IBertForMultipleChoice"),
+        ("longformer", "LongformerForMultipleChoice"),
         ("luke", "LukeForMultipleChoice"),
         ("megatron-bert", "MegatronBertForMultipleChoice"),
         ("mobilebert", "MobileBertForMultipleChoice"),
@@ -971,6 +995,7 @@
         ("distilbert", "DistilBertModel"),
         ("emu3", "Emu3TextModel"),
         ("ibert", "IBertModel"),
+        ("longformer", "LongformerModel"),
         ("mllama", "MllamaTextModel"),
         ("mobilebert", "MobileBertModel"),
         ("mt5", "MT5EncoderModel"),

mindone/transformers/models/longformer/__init__.py

@@ -0,0 +1,17 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# This code is adapted from https://github.com/huggingface/transformers
+# with modifications to run transformers on mindspore.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .modeling_longformer import *