feat(clf): enable opt out of adapter

modeling_seamless_m4t_v2_speech_encoder.py

@@ -74,7 +74,7 @@ class SeamlessM4Tv2ForAudioClassification(SeamlessM4Tv2PreTrainedModel):
             **kwargs,
         )
         hidden_states = outputs.last_hidden_state
-        if attention_mask is not None:
+        if attention_mask is not None and self.model.config.add_adapter:
             sub_sampled_lengths = self._compute_sub_sample_lengths_from_attention_mask(
                 attention_mask
             ).to(outputs.last_hidden_state.device)