Update configuration_progen.py

configuration_progen.py

@@ -29,7 +29,7 @@ class ProGenConfig(PretrainedConfig):
         vocab_size_emb=32,
         vocab_size_lm_head=32,
         n_positions=1024,
-        n_embd=1024,
+        embed_dim=1024,
         n_layer=12,
         n_head=16,
         rotary_dim=32,
@@ -49,13 +49,13 @@ class ProGenConfig(PretrainedConfig):
     ):
         super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
 
-        self.vocab_size_emb = vocab_size_emb
-        self.vocab_size_lm_head = vocab_size_lm_head
-        self.n_positions = n_positions      # context window size
-        self.n_embd = n_embd
+        self.vocab_size_emb = vocab_size_emb                # input vocab size
+        self.vocab_size_lm_head = vocab_size_lm_head        # output vocab size
+        self.n_positions = n_positions                      # context window size
+        self.embed_dim = embed_dim
         self.n_layer = n_layer
         self.n_head = n_head
-        self.n_inner = n_inner
+        self.n_inner = n_inner                              # inner dimension of the MLP
         self.rotary_dim = rotary_dim
         self.activation_function = activation_function
         self.resid_pdrop = resid_pdrop
@@ -70,18 +70,3 @@ class ProGenConfig(PretrainedConfig):
         self.bos_token_id = bos_token_id
         self.eos_token_id = eos_token_id
 
-    @property
-    def max_position_embeddings(self):
-        return self.n_positions
-
-    @property
-    def hidden_size(self):
-        return self.n_embd
-
-    @property
-    def num_attention_heads(self):
-        return self.n_head
-
-    @property
-    def num_hidden_layers(self):
-        return self.n_layer