sigmoid,swish:ok

Author: miaobyte

excuter/op-mem-ompsimd/src/deepx/op/init.hpp

@@ -19,15 +19,13 @@ namespace deepx::op{
         }
         void forward(mem::Mem &mem) override{
             auto output = mem.gettensor<T>(this->returns[0]).get();
-            if (is_float(this->args[0])){
-                T low = std::stof(this->args[0]);
-                T high = std::stof(this->args[1]);
-                tensorfunc::uniform(*output,low,high);
-            }else{
-                T low = mem.getarg<T>(this->args[0]);
-                T high = mem.getarg<T>(this->args[1]);
-                tensorfunc::uniform(*output,low,high);
+            T low =  this->getarg<T>(0,mem);
+            T high = this->getarg<T>(1,mem);
+            uint32_t seed = 0;
+            if (this->args.size() == 3){
+                seed = this->getarg<uint32_t>(2,mem);
             }
+            tensorfunc::uniform(*output,low,high,seed);
         } 
         void backward(mem::Mem &mem) override{
             throw std::runtime_error("Uniform op does not support backward");

excuter/op-mem-ompsimd/src/deepx/tensorfunc/init.hpp

@@ -10,7 +10,7 @@
 namespace deepx::tensorfunc
 {
     template <typename T>
-    void uniform(Tensor<T> &tensor,const T low = 0,const T high = 1)
+    void uniform(Tensor<T> &tensor, const T low = 0, const T high = 1, const unsigned int seed = 0)
     {
         std::uniform_real_distribution<double> distribution(low, high);
         std::random_device rd;
@@ -20,7 +20,17 @@ namespace deepx::tensorfunc
         std::vector<std::default_random_engine> generators(num_threads);
         for (int i = 0; i < num_threads; ++i)
         {
-            generators[i].seed(rd());
+            if (seed == 0)
+            {
+                // 使用随机设备生成种子
+                std::random_device rd;
+                generators[i].seed(rd());
+            }
+            else
+            {
+                // 使用主seed和线程ID生成确定性种子
+                generators[i].seed(seed + i);
+            }
         }
 
 #pragma omp parallel for
@@ -32,18 +42,16 @@ namespace deepx::tensorfunc
     }
 
     template <typename T>
-    void constant(Tensor<T> &tensor,const T value)
+    void constant(Tensor<T> &tensor, const T value)
     {
         std::fill(tensor.data, tensor.data + tensor.shape.size, value);
     }
 
     template <typename T>
-    void arange(Tensor<T> &tensor, const T start,const T step = 1)
+    void arange(Tensor<T> &tensor, const T start, const T step = 1)
     {
         tensor.shape.rangeParallel(tensor.shape.dim, [&](int idx_linear)
-                                   {
-                                       tensor.data[idx_linear] = start + (idx_linear)*step;
-                                   });
+                                   { tensor.data[idx_linear] = start + (idx_linear)*step; });
     }
 
     // template <typename T>

front/py/deepx/nn/functional/activite.py

@@ -36,7 +36,7 @@ def sigmoid(
         else:
             outtensor=out
     from .elementwise import exp
-    outtensor=1/((t*(-1)+1).exp())
+    outtensor = 1 / ((t*-1).exp()+1)
     return outtensor
 
 def swish(
@@ -56,32 +56,3 @@ def swish(
         输出张量
     """
     return x*sigmoid(x*beta,out=out)
-
-def swiglu(
-        x: Tensor,
-        W: Tensor,  # 第一个投影矩阵
-        V: Tensor,  # 第二个投影矩阵
-        beta: float = 1.0,  # swish函数的缩放因子
-        out: Union[Tensor,str] = '') -> Tensor:
-    """SwiGLU激活函数
-    
-    .. math::
-        \text{SwiGLU}(x, W, V) = \text{swish}(xW) \odot (xV)
-
-    其中:
-        - :math:`\odot` 表示逐元素乘法
-        - :math:`\text{swish}(x)` 是swish激活函数
-        - :math:`W` 和 :math:`V` 是投影矩阵
-
-    Args:
-        x: 输入张量
-        W: 第一个投影矩阵
-        V: 第二个投影矩阵  
-        beta: swish函数的缩放因子
-        out: 输出张量或名称
-
-    Returns:
-        输出张量
-    """
-    result=swish(x@W,beta=beta).mul(x@V,out=out)       
-    return result

front/py/deepx/nn/functional/elementwise.py

@@ -95,7 +95,7 @@ def sub(
     if isinstance(b,Tensor):
         return _A_B_elementwiseop_C(a,b,"sub",out)
     else:
-        return _A_b_elementwiseop_C(a,b,"sub_scalar",out)
+        return _A_b_elementwiseop_C(a,b*-1,"add_scalar",out)
 
 #mul
 OpNode.register("mul")

front/py/deepx/nn/functional/init.py

@@ -54,20 +54,21 @@ def arange(start=0, end=None, step=1,dtype=None, device=None,name:Union[Tensor,s
     return outtensor
 
 OpNode.register("uniform")
-def uniform(t:Tensor,low=0, high=1)->Tensor:
+def uniform(t:Tensor,low=0, high=1,seed:int=0)->Tensor:
     if low >= high:
         raise ValueError(f"low({low})必须小于high({high})")
     if t is None:
         raise ValueError("t不能为None")
     g=t.graph
-    arglow=g.add_var('',low)
-    arghigh=g.add_var('',high)
+ 
     opnode = g.add_op("uniform")
-    opnode.add_input(arglow)
-    opnode.add_input(arghigh)
+    opnode.add_input(g.add_var('',low))
+    opnode.add_input(g.add_var('',high))
+    if seed is not None:
+        opnode.add_input(g.add_var('',seed))
     t.node.add_input(opnode)
     if t.graph.eager:
-        ir=DeepxIR("uniform", t.dtype, [low, high], [t.node.name])
+        ir=DeepxIR("uniform", t.dtype, [low, high,seed], [t.node.name])
         send(ir)
     return t
 

front/py/deepx/nn/modules/activation.py

@@ -1,4 +1,5 @@
-from deepx.tensor import Tensor
+from typing import Union
+from deepx  import Tensor,ones
 import deepx.nn.functional as F
 from .module import Module
 
@@ -24,4 +25,32 @@ def __init__(self):
 
     def forward(self, input: Tensor) -> Tensor:
         return F.sigmoid(input)
-    
+    
+class Swish(Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, input: Tensor) -> Tensor:
+        return F.swish(input)
+
+ 
+class Swiglu(Module):
+    def __init__(self):
+        super().__init__()
+        self.W = ones(shape=(1,1),name=self.full_name+"_W")
+        self.V = ones(shape=(1,1),name=self.full_name+"_V")
+
+    def swiglu(
+        x: Tensor,
+        W: Tensor,  # 第一个投影矩阵
+        V: Tensor,  # 第二个投影矩阵
+        beta: float = 1.0,  # swish函数的缩放因子
+        out: Union[Tensor,str] = '') -> Tensor:
+        from deepx.nn.functional import swish
+        result=swish(x@W,beta=beta).mul(x@V,out=out)       
+        return result
+ 
+    
+    def forward(self, input: Tensor) -> Tensor:
+        return self.swiglu(input,self.W,self.V)
+ 

front/py/deepx/tensor/init.py

@@ -16,9 +16,9 @@ def ones_(self):
     constant_func(self,value=1)
 
 @tensor_method
-def uniform_(self,low=0, high=1):
+def uniform_(self,low=0, high=1,seed:int=0):
     from deepx.nn.functional import uniform as uniform_func
-    uniform_func(self,low=low, high=high)
+    uniform_func(self,low=low, high=high,seed=seed)
 
 @tensor_method
 def rand_(self):

front/py/examples/3_functional/1_sigmoid.dot

@@ -2,45 +2,55 @@
 digraph {
 	rankdir=TB
 	node [shape=record]
-	124458605174432 [label="x
+	130115172244864 [label="x
 (3, 4, 5)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605170256 [label="var_1
--1" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605174000 [label="var_2
-1" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605169248 [label=uniform color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
-	124458605656912 [label="out
-(3, 4, 5)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605657104 [label="tensor_3
+	130117430101712 [label=reshape color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
+	130117430101376 [label="vector_1
+(3, 4, 5)" color=darkseagreen fillcolor=honeydew fontname="Sans-Serif" labeljust=l shape=box style=filled]
+	130115170569184 [label=div_scalar color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
+	130117430113280 [label="var_1
+10.0" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled]
+	130115170569280 [label=add_scalar color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
+	130115170569136 [label="var_2
+-3.0" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled]
+	130115170569088 [label="out
 (3, 4, 5)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605657200 [label=mul_scalar color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
-	124458605657152 [label="var_3
+	130115170569520 [label=mul_scalar color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
+	130115170569472 [label="var_3
 -1" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605657488 [label="tensor_4
+	130115170569664 [label="tensor_3
 (3, 4, 5)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605657584 [label=add_scalar color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
-	124458605657536 [label="var_4
-1" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605657872 [label="tensor_5
+	130115170569904 [label=exp color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
+	130115170570000 [label="tensor_4
 (3, 4, 5)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605657968 [label=exp color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
-	124458605904032 [label="tensor_6
+	130115170576384 [label=add_scalar color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
+	130115170576336 [label="var_4
+1" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled]
+	130115170576528 [label="tensor_5
 (3, 4, 5)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605904128 [label=rdiv_scalar color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
-	124458605904080 [label="var_5
+	130115170576768 [label=rdiv_scalar color=darkslategray fillcolor=lightgray fontname="Courier Bold" labeljust=l shape=box style=filled]
+	130115170576720 [label="var_5
 1" color=orange fillcolor=moccasin fontname="Sans-Serif" labeljust=l shape=box style=filled]
-	124458605169248 -> 124458605174432 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605170256 -> 124458605169248 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605174000 -> 124458605169248 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605657200 -> 124458605657104 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605174432 -> 124458605657200 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605657152 -> 124458605657200 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605657584 -> 124458605657488 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605657104 -> 124458605657584 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605657536 -> 124458605657584 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605657968 -> 124458605657872 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605657488 -> 124458605657968 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605904128 -> 124458605904032 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605904080 -> 124458605904128 [arrowsize=0.8 color=gray40 penwidth=1.2]
-	124458605657872 -> 124458605904128 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170576912 [label="tensor_6
+(3, 4, 5)" color=skyblue fillcolor=aliceblue fontname="Sans-Serif" labeljust=l shape=box style=filled]
+	130117430101712 -> 130115172244864 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170569184 -> 130115172244864 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170569280 -> 130115172244864 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115172244864 -> 130117430101712 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130117430101376 -> 130117430101712 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115172244864 -> 130115170569184 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130117430113280 -> 130115170569184 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115172244864 -> 130115170569280 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170569136 -> 130115170569280 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115172244864 -> 130115170569520 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170569472 -> 130115170569520 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170569520 -> 130115170569664 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170569664 -> 130115170569904 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170569904 -> 130115170570000 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170570000 -> 130115170576384 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170576336 -> 130115170576384 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170576384 -> 130115170576528 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170576720 -> 130115170576768 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170576528 -> 130115170576768 [arrowsize=0.8 color=gray40 penwidth=1.2]
+	130115170576768 -> 130115170576912 [arrowsize=0.8 color=gray40 penwidth=1.2]
 }