relu:实现

Author: miaobyte

excuter/op-mem-ompsimd/src/deepx/op/opfactory.cpp

@@ -17,7 +17,6 @@ namespace deepx::op
         opfactory.add_op(NewTensor<int64_t>());
         opfactory.add_op(NewTensor<float>());
         opfactory.add_op(NewTensor<double>());
-
         opfactory.add_op(ArgSet<int32_t>());
         opfactory.add_op(ArgSet<float>());
         opfactory.add_op(ArgSet<double>());
@@ -124,6 +123,7 @@ namespace deepx::op
         register_new(opfactory);
         register_init(opfactory);
         register_print(opfactory);
+        register_transpose(opfactory);
         register_elementwise_op(opfactory);
         register_concat(opfactory);
         register_matmul(opfactory);

front/py/deepx/__init__.py

@@ -1,20 +1,23 @@
 from .tensor import Tensor,Shape,Device,DeviceType
-from deepx.nn.functional import full,zeros,ones,arange,rand,randn,eye
-from deepx.nn.functional import add,sub,mul,div,clamp
-from deepx.nn.functional import matmul
-from deepx.nn.functional import max,min,sum,prod,mean
+from deepx.nn.functional import *
 __all__ = [
+    #tensor
     'Tensor',
     'Shape',
     'Device','DeviceType',
-    #init
-    'full','zeros', 'ones', 'arange', 'rand', 'randn', 'eye',
-    #elementwise
-    "add","sub","mul","div","clamp",
-    #matmul
-    "matmul",
-    #reduce
-    "max","min","sum","prod","mean",
+    #nn.functional
+        #init
+        'full','zeros', 'ones', 'arange', 'rand', 'randn', 'eye',
+        #elementwise
+        "add","sub","mul","div","clamp",
+        #matmul
+        "matmul",
+        #reduce
+        "max","min","sum","prod","mean",
+        #transpose
+        "transpose",
+        #relu
+        "relu",
 ]
 
 # 为了支持 import deepx as dx 的用法

front/py/deepx/nn/functional/__init__.py

@@ -5,6 +5,7 @@
 from .init import constant,full,zeros,ones,arange,rand,randn,eye
 from .reduce import max,min,sum,prod,mean
 from .transpose import transpose
+from .activite import relu
 
 __all__ = [
     "newtensor",
@@ -14,4 +15,5 @@
     "matmul",
     "max","min","sum","prod","mean",
     "transpose",    
+    "relu",
 ]

front/py/deepx/nn/functional/activite.py

@@ -0,0 +1,12 @@
+from deepx.tensor import Tensor
+from deepx.nn.deepxir import DeepxIR
+from deepx.scheduler import send
+
+def relu(t: Tensor,inplace:bool=False)->Tensor:
+    out=t
+    if not inplace:
+        out=Tensor(shape=t.shape, dtype=t.dtype, device=t.device)
+    ir=DeepxIR("max_scalar",t.dtype,[t._node.name,0], [out._node.name])
+    send(str(ir))
+    return out
+ 

front/py/deepx/nn/functional/transpose.py

@@ -5,7 +5,11 @@
 def transpose(t: Tensor,dimorder:list[int]=None,out:Tensor=None):
     if dimorder is None:
         dimorder=list(range(t.ndimension))
-    ir=DeepxIR("transpose",'any',[t._node.name,*map(str, dimorder)], [out._node.name])
+ 
+    if out is None:
+        out=Tensor(shape=t.Shape.transpose(dimorder), dtype=t.dtype, device=t.device)
+    ir=DeepxIR("transpose",'',[t._node.name,*map(str, dimorder)], [out._node.name])
     send(str(ir))
+
     return out
 

front/py/deepx/nn/modules/activation.py

@@ -0,0 +1,46 @@
+from deepx.tensor import Tensor
+import deepx.nn.functional as F
+from .module import Module
+
+#copy from pytorch
+class ReLU(Module):
+    r"""Applies the rectified linear unit function element-wise.
+
+    :math:`\text{ReLU}(x) = (x)^+ = \max(0, x)`
+
+    Args:
+        inplace: can optionally do the operation in-place. Default: ``False``
+
+    Shape:
+        - Input: :math:`(*)`, where :math:`*` means any number of dimensions.
+        - Output: :math:`(*)`, same shape as the input.
+
+    .. image:: ../scripts/activation_images/ReLU.png
+
+    Examples::
+
+        >>> m = nn.ReLU()
+        >>> input = torch.randn(2)
+        >>> output = m(input)
+
+
+      An implementation of CReLU - https://arxiv.org/abs/1603.05201
+
+        >>> m = nn.ReLU()
+        >>> input = torch.randn(2).unsqueeze(0)
+        >>> output = torch.cat((m(input), m(-input)))
+    """
+
+    __constants__ = ["inplace"]
+    inplace: bool
+
+    def __init__(self, inplace: bool = False):
+        super().__init__()
+        self.inplace = inplace
+
+    def forward(self, input: Tensor) -> Tensor:
+        return F.relu(input, inplace=self.inplace)
+
+    def extra_repr(self) -> str:
+        inplace_str = "inplace=True" if self.inplace else ""
+        return inplace_str

front/py/deepx/tensor/shape.py

@@ -85,4 +85,8 @@ def __hash__(self):
         """使Shape可哈希，便于在字典和集合中使用"""
         return hash(self.shape)
 
+    def transpose(self,dimorder:list[int]=None):
+        if dimorder is None:
+            dimorder=list(range(self.ndimension))
+        return Shape(tuple(self.shape[i] for i in dimorder))
 

front/py/deepx/tensor/tensor.py

@@ -54,6 +54,9 @@ def __init__(
     @property
     def shape(self):
         return self._shape.shape
+    @property
+    def Shape(self):
+        return self._shape
 
     @property
     def stride(self):

front/py/deepx/tensor/transpose.py

@@ -1,14 +1,13 @@
 from  .tensor import Tensor,tensor_method
 
 @tensor_method
-def transpose(self,dimorder:list[int]=None, out:Tensor=None):
-    result = Tensor(dtype=self.dtype,shape=self.shape)
+def transpose(self,*axes):
     from deepx.nn.functional import transpose as transpose_func
-    transpose_func(self,dimorder,result)
+    result=transpose_func(self,axes)
     return result
 
 @tensor_method
-def transpose_(self,dimorder:list[int]=None):
+def transpose_(self,*axes):
     from deepx.nn.functional import transpose as transpose_func
-    transpose_func(self,dimorder,self)
+    transpose_func(self,axes,self)
     return self

front/py/examples/2_ir/4_transpose.py

@@ -3,5 +3,5 @@
 print()
 
 t1 = ones([3,4],dtype='float32')
-t2=t1.transpose(dimorder=[1,0])
+t2=t1.transpose(1,0)
 print(t2)