feat(transformers): add flava model

mindone/transformers/__init__.py

@@ -513,6 +513,18 @@
     FlaubertPreTrainedModel,
     FlaubertWithLMHeadModel,
 )
+from .models.flava import (
+    FlavaFeatureExtractor,
+    FlavaForPreTraining,
+    FlavaImageCodebook,
+    FlavaImageModel,
+    FlavaImageProcessor,
+    FlavaModel,
+    FlavaMultimodalModel,
+    FlavaPreTrainedModel,
+    FlavaProcessor,
+    FlavaTextModel,
+)
 from .models.fnet import (
     FNetForMaskedLM,
     FNetForMultipleChoice,

mindone/transformers/models/__init__.py

@@ -78,6 +78,7 @@
     falcon_mamba,
     fastspeech2_conformer,
     flaubert,
+    flava,
     fnet,
     focalnet,
     fsmt,

mindone/transformers/models/auto/configuration_auto.py

@@ -100,6 +100,7 @@
         ("falcon", "FalconConfig"),
         ("falcon_mamba", "FalconMambaConfig"),
         ("fastspeech2_conformer", "FastSpeech2ConformerConfig"),
+        ("flava", "FlavaConfig"),
         ("fnet", "FNetConfig"),
         ("focalnet", "FocalNetConfig"),
         ("funnel", "FunnelConfig"),
@@ -154,6 +155,7 @@
         ("luke", "LukeConfig"),
         ("mamba", "MambaConfig"),
         ("mamba2", "Mamba2Config"),
+        ("mbart", "MBartConfig"),
         ("mimi", "MimiConfig"),
         ("mistral", "MistralConfig"),
         ("mistral3", "Mistral3Config"),
@@ -372,6 +374,7 @@
         ("falcon", "Falcon"),
         ("falcon_mamba", "FalconMamba"),
         ("fastspeech2_conformer", "FastSpeech2Conformer"),
+        ("flava", "FLAVA"),
         ("fnet", "FNet"),
         ("focalnet", "FocalNet"),
         ("fsmt", "FairSeq Machine-Translation"),

mindone/transformers/models/auto/feature_extraction_auto.py

@@ -46,6 +46,7 @@
         ("chinese_clip", "ChineseCLIPFeatureExtractor"),
         ("convnext", "ConvNextFeatureExtractor"),
         ("cvt", "ConvNextFeatureExtractor"),
+        ("flava", "FlavaFeatureExtractor"),
         ("seamless_m4t", "SeamlessM4TFeatureExtractor"),
     ]
 )

mindone/transformers/models/auto/image_processing_auto.py

@@ -61,6 +61,7 @@
             ("dinov2", ("BitImageProcessor",)),
             ("dpt", ("DPTImageProcessor",)),
             ("efficientnet", ("EfficientNetImageProcessor",)),
+            ("flava", ("FlavaImageProcessor",)),
             ("oneformer", ("OneFormerImageProcessor",)),
             ("llava_next", ("LlavaNextImageProcessor",)),
             ("llava_next_video", ("LlavaNextVideoImageProcessor",)),

mindone/transformers/models/auto/modeling_auto.py

@@ -99,6 +99,7 @@
         ("falcon", "FalconModel"),
         ("falcon_mamba", "FalconMambaModel"),
         ("fastspeech2_conformer", "FastSpeech2ConformerModel"),
+        ("flava", "FlavaModel"),
         ("fnet", "FNetModel"),
         ("focalnet", "FocalNetModel"),
         ("fsmt", "FSMTModel"),
@@ -282,9 +283,10 @@
         ("ctrl", "CTRLLMHeadModel"),
         ("data2vec-text", "Data2VecTextForMaskedLM"),
         ("distilbert", "DistilBertForMaskedLM"),
-        ("fnet", "FNetForPreTraining"),
         ("electra", "ElectraForPreTraining"),
         ("falcon_mamba", "FalconMambaForCausalLM"),
+        ("flava", "FlavaForPreTraining"),
+        ("fnet", "FNetForPreTraining"),
         ("fsmt", "FSMTForConditionalGeneration"),
         ("funnel", "FunnelForPreTraining"),
         ("gpt2", "GPT2LMHeadModel"),

mindone/transformers/models/auto/processing_auto.py

@@ -53,6 +53,7 @@
         ("chameleon", "ChameleonProcessor"),
         ("chinese_clip", "ChineseCLIPProcessor"),
         ("colpali", "ColPaliProcessor"),
+        ("flava", "FlavaProcessor"),
         ("idefics", "IdeficsProcessor"),
         ("layoutlmv3", "LayoutMv3Processor"),
         ("instructblip", "InstructBlipProcessor"),

mindone/transformers/models/flava/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2024 The HuggingFace Team. All rights reserved.
+#
+# This code is adapted from https://github.com/huggingface/transformers
+# with modifications to run transformers on mindspore.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .feature_extraction_flava import *
+from .image_processing_flava import *
+from .modeling_flava import *
+from .processing_flava import *

mindone/transformers/models/flava/feature_extraction_flava.py

@@ -0,0 +1,38 @@
+# coding=utf-8
+# Copyright 2022 Meta Platforms authors and The HuggingFace Team. All rights reserved.
+#
+# This code is adapted from https://github.com/huggingface/transformers
+# with modifications to run transformers on mindspore.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Feature extractor class for FLAVA."""
+
+import warnings
+
+from ...utils import logging
+from .image_processing_flava import FlavaImageProcessor
+
+logger = logging.get_logger(__name__)
+
+
+class FlavaFeatureExtractor(FlavaImageProcessor):
+    def __init__(self, *args, **kwargs) -> None:
+        warnings.warn(
+            "The class FlavaFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"
+            " use FlavaImageProcessor instead.",
+            FutureWarning,
+        )
+        super().__init__(*args, **kwargs)
+
+
+__all__ = ["FlavaFeatureExtractor"]