rope:实现中，_compute_llama3_parameters

Author: miaobyte

doc/excuter/op-mem-cuda/list.md

@@ -65,7 +65,7 @@
 | cos | miaobyte | T3=cos(T1) | cos(tensor<float64|float32|float16|bfloat16> A)->(tensor<float64|float32|float16|bfloat16> C) |
 | notequalscalar | miaobyte | T1!=scalar->mask | notequalscalar(tensor<any> A, var<any> scalar, var<float32> epsilon)->(tensor<bool> mask) |
 | minscalar | miaobyte | T3=min(T1, scalar) | minscalar(tensor<any> A, var<any> scalar)->(tensor<any> C) |
-| rpowscalar | miaobyte | T3=pow(scalar, T1) | rpowscalar(var<float64|int32> scalar, tensor<float64|float32> A)->(tensor<float64|float32> C) |
+| rpowscalar | miaobyte | T3=pow(scalar, T1) | rpowscalar(var<float32|int32> scalar, tensor<float64|float32> A)->(tensor<float64|float32> C) |
 | rdivscalar | miaobyte | T3=scalar/T1 | rdivscalar(var<any> scalar, tensor<any> A)->(tensor<any> C) |
 | less | miaobyte | mask=compare(T1, T2) | less(tensor<any> A, tensor<any> B)->(tensor<bool> mask) |
 | powscalar | miaobyte | T3=pow(T1, scalar) | powscalar(tensor<float64|float32> A, var<float64|int32> scalar)->(tensor<float64|float32> C) |

excuter/op-mem-cuda/src/client/tfs.cpp

@@ -317,7 +317,7 @@ namespace deepx::tf
         // rpowscalar
         tffactory.add_tf(std::make_shared<RpowScalar<miaobyte>>(vector<Param>(
                                                                     {
-                                                                        Param("scalar", DataCategory::Var, Precision::Float64 | Precision::Int32),
+                                                                        Param("scalar", DataCategory::Var, Precision::Float32 | Precision::Int32),
                                                                         Param("A", DataCategory::Tensor, Precision::Float64 | Precision::Float32),
                                                                     }),
                                                                 vector<Param>(

excuter/op-mem-ompsimd/src/client/tfs.cpp

@@ -338,7 +338,7 @@ namespace deepx::tf
         // rpowscalar author=miaobyte
         tffactory.add_tf(std::make_shared<RpowScalar<miaobyte>>(vector<Param>(
                                                                     {
-                                                                        Param("scalar", DataCategory::Var, Precision::Any),
+                                                                        Param("scalar", DataCategory::Var, Precision::Float32),
                                                                         Param("A", DataCategory::Tensor, Precision::Any),
                                                                     }),
                                                                 vector<Param>(

front/py/deepx/transformer/modeling_rope_utils.py

@@ -26,12 +26,15 @@ def _compute_llama3_parameters(config:dict={
 }) -> Tuple[Tensor, float]:
     # Gets the default RoPE parameters
     inv_freq, attention_factor = _compute_default_rope_parameters(config)
-    #TODO
 
-    low_freq_wavelen = config.old_context_len / config.low_freq_factor
-    high_freq_wavelen = config.old_context_len / config.high_freq_factor
+    low_freq_factor = config["rope_scaling"]["low_freq_factor"]  # `1` in the original implementation
+    high_freq_factor = config["rope_scaling"]["high_freq_factor"]  # `4` in the original implementation
+    old_context_len = config["rope_scaling"]["original_max_position_embeddings"]  # `8192` in the original implementation
+    low_freq_wavelen = old_context_len /low_freq_factor
+    high_freq_wavelen = old_context_len/ high_freq_factor
 
     wavelen = 2 * math.pi / inv_freq
+    wavelen.print()
     # wavelen < high_freq_wavelen: do nothing
     # wavelen > low_freq_wavelen: divide by factor
     inv_freq_llama =  where(wavelen > low_freq_wavelen, inv_freq / config.factor, inv_freq)