diff --git a/doc/design.md b/doc/design.md index bbc4df71..0c93fe33 100644 --- a/doc/design.md +++ b/doc/design.md @@ -18,4 +18,11 @@ deepIR{ excuter执行deepxIR的规则 + excuter执行deepxIR时，不得修改args中的tensor -+ 但deepIR不限制args和returns中的Param同名，这样可以实现类似inplace的操作 \ No newline at end of file ++ 但deepIR不限制args和returns中的Param同名，这样可以实现类似inplace的操作 + + +## front/python规则 + +### 1.命名规则 ++ inplace操作的函数，其名为_后缀, 返回值为空 ++ 非inplace操作的函数，其名无_后缀 diff --git a/doc/excuter/op-mem-cuda/list.md b/doc/excuter/op-mem-cuda/list.md index b177d4e9..475d0f43 100644 --- a/doc/excuter/op-mem-cuda/list.md +++ b/doc/excuter/op-mem-cuda/list.md @@ -48,7 +48,6 @@ | Operation | Author | Math Formula | IR Instruction | |-----------|--------|--------------|----------------| -| dropout | miaobyte | T1.dropout(p,seed)->T3 | dropout(tensor A, var p, var seed)->(tensor C) | | switch | miaobyte | C=switch(tensors,cases) | switch(listtensor tensors, tensor cases)->(tensor result) | | greaterscalar | miaobyte | mask=compare(T1, scalar) | greaterscalar(tensor A, var scalar)->(tensor mask) | | equalscalar | miaobyte | mask=compare(T1, scalar) | equalscalar(tensor A, var scalar, var epsilon)->(tensor mask) | @@ -56,17 +55,18 @@ | maxscalar | miaobyte | T3=max(T1, scalar) | maxscalar(tensor A, var scalar)->(tensor C) | | tan | miaobyte | T3=tan(T1) | tan(tensor A)->(tensor C) | | sin | miaobyte | T3=sin(T1) | sin(tensor A)->(tensor C) | +| dropout | miaobyte | dropout(p,seed)->A | dropout(var p, var seed)->(tensor A) | | divscalar | miaobyte | T3=scalar/T1 | divscalar(tensor A, var scalar)->(tensor C) | | log | miaobyte | T3=log(T1) | log(tensor A)->(tensor C) | | addscalar | miaobyte | T3=T1+scalar | addscalar(tensor A, var b)->(tensor C) | | greater | miaobyte | mask=compare(T1, T2) | greater(tensor A, tensor B)->(tensor mask) | | lessscalar | miaobyte | mask=compare(T1, scalar) | lessscalar(tensor A, var scalar)->(tensor mask) | | cos | miaobyte | T3=cos(T1) | cos(tensor A)->(tensor C) | -| less | miaobyte | mask=compare(T1, T2) | less(tensor A, tensor B)->(tensor mask) | -| powscalar | miaobyte | T3=pow(T1, scalar) | powscalar(tensor A, var scalar)->(tensor C) | | minscalar | miaobyte | T3=min(T1, scalar) | minscalar(tensor A, var scalar)->(tensor C) | -| rdivscalar | miaobyte | T3=scalar/T1 | rdivscalar(var scalar, tensor A)->(tensor C) | | rpowscalar | miaobyte | T3=pow(scalar, T1) | rpowscalar(var scalar, tensor A)->(tensor C) | +| rdivscalar | miaobyte | T3=scalar/T1 | rdivscalar(var scalar, tensor A)->(tensor C) | +| less | miaobyte | mask=compare(T1, T2) | less(tensor A, tensor B)->(tensor mask) | +| powscalar | miaobyte | T3=pow(T1, scalar) | powscalar(tensor A, var scalar)->(tensor C) | | todtype | none | T3(dtypeA)->T1(dtypeB) | todtype(tensor a)->(tensor b) | | add | cublas | T3=T1+T2 | add(tensor a, tensor b)->(tensor c) | | add | miaobyte | T3=T1+T2 | add(tensor a, tensor b)->(tensor c) | diff --git a/doc/excuter/op-mem-ompsimd/list.md b/doc/excuter/op-mem-ompsimd/list.md index 58772944..35a3a99e 100644 --- a/doc/excuter/op-mem-ompsimd/list.md +++ b/doc/excuter/op-mem-ompsimd/list.md @@ -49,22 +49,22 @@ | Operation | Author | Math Formula | IR Instruction | |-----------|--------|--------------|----------------| -| dropout | miaobyte | A.dropout(p,seed)->C | dropout(tensor A, var p, var seed)->(tensor C) | | switch | miaobyte | C=switch([tensors],case) | switch(listtensor tensors, tensor cases)->(tensor C) | | greaterscalar | miaobyte | mask=greater(T1,scalar) | greaterscalar(tensor A, var scalar)->(tensor mask) | | equalscalar | miaobyte | mask=equal(T1,scalar) | equalscalar(tensor A, var scalar)->(tensor mask) | | min | miaobyte | T3=min(T1,T2) | min(tensor A, tensor B)->(tensor C) | | maxscalar | miaobyte | T3=max(T1,scalar) | maxscalar(tensor A, var scalar)->(tensor C) | +| dropout | miaobyte | dropout(p,seed)->A | dropout(var p, var seed)->(tensor A) | | divscalar | miaobyte | T3=T1/scalar | divscalar(tensor A, var scalar)->(tensor C) | | log | miaobyte | T3=log(T1) | log(tensor A)->(tensor C) | | addscalar | miaobyte | T3=T1+scalar | addscalar(tensor a, var scalar)->(tensor c) | | greater | miaobyte | mask=greater(T1,T2) | greater(tensor A, tensor B)->(tensor mask) | | lessscalar | miaobyte | mask=less(T1,scalar) | lessscalar(tensor A, var scalar)->(tensor mask) | -| less | miaobyte | mask=less(T1,T2) | less(tensor A, tensor B)->(tensor mask) | -| powscalar | miaobyte | T3=T1^scalar | powscalar(tensor A, var scalar)->(tensor C) | | minscalar | miaobyte | T3=min(T1,scalar) | minscalar(tensor A, var scalar)->(tensor C) | -| rdivscalar | miaobyte | T3=scalar/T1 | rdivscalar(var scalar, tensor A)->(tensor C) | | rpowscalar | miaobyte | T3=scalar^T1 | rpowscalar(var scalar, tensor A)->(tensor C) | +| rdivscalar | miaobyte | T3=scalar/T1 | rdivscalar(var scalar, tensor A)->(tensor C) | +| less | miaobyte | mask=less(T1,T2) | less(tensor A, tensor B)->(tensor mask) | +| powscalar | miaobyte | T3=T1^scalar | powscalar(tensor A, var scalar)->(tensor C) | | todtype | none | T3(dtypeA)->T1(dtypeB) | todtype(tensor A)->(tensor C) | | add | cblas | T3=T1+T2 | add(tensor a, tensor b)->(tensor c) | | add | miaobyte | T3=T1+T2 | add(tensor a, tensor b)->(tensor c) | diff --git a/excuter/cpp-common/src/deepx/tensorfunc/elementwise.hpp b/excuter/cpp-common/src/deepx/tensorfunc/elementwise.hpp index 52fd04bc..eb63f674 100644 --- a/excuter/cpp-common/src/deepx/tensorfunc/elementwise.hpp +++ b/excuter/cpp-common/src/deepx/tensorfunc/elementwise.hpp @@ -410,20 +410,7 @@ namespace deepx::tensorfunc void invert(const Tensor &input, Tensor &output) { invertDispatcher::invert(input, output); - } - - //dropout(A,p)=>C - template - struct dropoutDispatcher - { - static void dropout(const Tensor &input, const float p,const unsigned int seed, Tensor &output) = delete; }; - - template - void dropout(const Tensor &input, const float p,const unsigned int seed, Tensor &output) - { - dropoutDispatcher::dropout(input, p, seed, output); - } } // namespace deepx::tensorfunc diff --git a/excuter/cpp-common/src/deepx/tensorfunc/init.hpp b/excuter/cpp-common/src/deepx/tensorfunc/init.hpp index fdca437b..934e25f1 100644 --- a/excuter/cpp-common/src/deepx/tensorfunc/init.hpp +++ b/excuter/cpp-common/src/deepx/tensorfunc/init.hpp @@ -19,6 +19,20 @@ namespace deepx::tensorfunc constantDispatcher::constant(tensor, value); } + //dropout(A,p)=>C + template + struct dropoutDispatcher + { + static void dropout(Tensor &input, const float p,const unsigned int seed) = delete; + }; + + template + void dropout(Tensor &input, const float p,const unsigned int seed) + { + dropoutDispatcher::dropout(input, p, seed); + } + + //arange template struct arangeDispatcher diff --git a/excuter/op-mem-cuda/src/client/tfs.cpp b/excuter/op-mem-cuda/src/client/tfs.cpp index 5e304d29..0c0ca44e 100644 --- a/excuter/op-mem-cuda/src/client/tfs.cpp +++ b/excuter/op-mem-cuda/src/client/tfs.cpp @@ -91,7 +91,17 @@ namespace deepx::tf vector({ Param("t", DataCategory::Tensor, Precision::Any), }))); - + // dropout + tffactory.add_tf(std::make_shared>(vector( + { + Param("p", DataCategory::Var, Precision::Float32), + Param("seed", DataCategory::Var, Precision::Int32), + }), + vector( + { + Param("A", DataCategory::Tensor, Precision::Any), + }))); + tffactory.add_tf(std::make_shared>(vector( { @@ -462,17 +472,7 @@ namespace deepx::tf { Param("result", DataCategory::Tensor, Precision::Any), }))); - // dropout - tffactory.add_tf(std::make_shared>(vector( - { - Param("A", DataCategory::Tensor, Precision::Any), - Param("p", DataCategory::Var, Precision::Float32), - Param("seed", DataCategory::Var, Precision::Int32), - }), - vector( - { - Param("C", DataCategory::Tensor, Precision::Any), - }))); + } // matmul void register_matmul(TfFactory &tffactory) diff --git a/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.cu b/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.cu index 8550e6b0..7ee88bb1 100644 --- a/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.cu +++ b/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.cu @@ -407,48 +407,6 @@ namespace deepx::tensorfunc template void launch_invert(const int16_t *a, int16_t *c, const int size); template void launch_invert(const int8_t *a, int8_t *c, const int size); - //dropout - template - __global__ void dropout_kernel(const T *A, const float p,const unsigned int seed, T *C, const int size) - { - int stride = blockDim.x * gridDim.x; - curandState state; - curand_init(seed, threadIdx.x, 0, &state); // 仅初始化一次 - - for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size; idx += stride) - { - float rand = curand_uniform(&state); - if (rand < p) - { - C[idx] = 0; - } - else - { - C[idx] = A[idx]; - } - } - } - - template - void launch_dropout(const T *a, const float p,const unsigned int seed, T *c, const int size) - { - auto [numBlocks, blockSize] = BestDims(size); - dropout_kernel<<>>(a, p, seed, c, size); - cudaError_t err = cudaGetLastError(); - if (err != cudaSuccess) - { - throw std::runtime_error("Failed to launch dropout kernel: " + - std::string(cudaGetErrorString(err))); - } - } - template void launch_dropout(const double *a, const float p,const unsigned int seed, double *c, const int size); - template void launch_dropout(const float *a, const float p,const unsigned int seed, float *c, const int size); - template void launch_dropout(const half *a, const float p,const unsigned int seed, half *c, const int size); - template void launch_dropout(const nv_bfloat16 *a, const float p,const unsigned int seed, nv_bfloat16 *c, const int size); - template void launch_dropout(const int64_t *a, const float p,const unsigned int seed, int64_t *c, const int size); - template void launch_dropout(const int32_t *a, const float p,const unsigned int seed, int32_t *c, const int size); - template void launch_dropout(const int16_t *a, const float p,const unsigned int seed, int16_t *c, const int size); - template void launch_dropout(const int8_t *a, const float p,const unsigned int seed, int8_t *c, const int size); } #endif // DEEPX_TENSORFUNC_ELEMENTWISE_MIAOBYTE_BASIC_CU diff --git a/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.cuh b/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.cuh index c85cce81..4100f38d 100644 --- a/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.cuh +++ b/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.cuh @@ -89,12 +89,6 @@ namespace deepx::tensorfunc template void launch_invert(const T* a, T* c,const int size); - //dropout - template - __global__ void dropout_kernel(const T* A, const float p,const unsigned int seed, T* C,const int size); - - template - void launch_dropout(const T* a, const float p,const unsigned int seed, T* c,const int size); } #endif // DEEPX_TENSORFUNC_ELEMENTWISE_MIAO_BYTE_BASIC_CUH diff --git a/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.hpp b/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.hpp index 65f7a82d..b7d3a680 100644 --- a/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.hpp +++ b/excuter/op-mem-cuda/src/deepx/tensorfunc/elementwise_miaobyte_basic.hpp @@ -140,15 +140,6 @@ namespace deepx::tensorfunc launch_invert( A.data, C.data, A.shape.size); } }; - - template - struct dropoutDispatcher - { - static void dropout(const Tensor &A, const float p,const unsigned int seed, Tensor &C) - { - launch_dropout(A.data, p, seed, C.data, A.shape.size); - } - }; } #endif // DEEPX_TENSORFUNC_ELEMENTWISE_MIAO_BYTE_BASIC_HPP diff --git a/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.cu b/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.cu index 82497ee5..7f8d2612 100644 --- a/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.cu +++ b/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.cu @@ -7,6 +7,8 @@ #include "deepx/tensorfunc/cuda.hpp" namespace deepx::tensorfunc { + // 填充 + // constant template __global__ void kernel_constant(T *data, const T value, const int size) { @@ -40,7 +42,47 @@ namespace deepx::tensorfunc template void launch_constant(int8_t *a, const int8_t value, const int size); template void launch_constant(bool *a, const bool value, const int size); - // 添加kernel函数 + // dropout + template + __global__ void dropout_kernel(T *A, const float p, const unsigned int seed, const int size) + { + int stride = blockDim.x * gridDim.x; + curandState state; + curand_init(seed, threadIdx.x, 0, &state); // 仅初始化一次 + + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size; idx += stride) + { + float rand = curand_uniform(&state); + if (rand < p) + { + A[idx] = 0; + } + } + } + + template + void launch_dropout(T *a, const float p, const unsigned int seed, const int size) + { + auto [numBlocks, blockSize] = BestDims(size); + dropout_kernel<<>>(a, p, seed, size); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + throw std::runtime_error("Failed to launch dropout kernel: " + + std::string(cudaGetErrorString(err))); + } + } + template void launch_dropout(double *a, const float p, const unsigned int seed, const int size); + template void launch_dropout(float *a, const float p, const unsigned int seed, const int size); + template void launch_dropout(half *a, const float p, const unsigned int seed, const int size); + template void launch_dropout(nv_bfloat16 *a, const float p, const unsigned int seed, const int size); + template void launch_dropout(int64_t *a, const float p, const unsigned int seed, const int size); + template void launch_dropout(int32_t *a, const float p, const unsigned int seed, const int size); + template void launch_dropout(int16_t *a, const float p, const unsigned int seed, const int size); + template void launch_dropout(int8_t *a, const float p, const unsigned int seed, const int size); + + // 初始化 + // arange template __global__ void kernel_arange(T *data, const float start, const float step, const int size) { @@ -133,7 +175,7 @@ namespace deepx::tensorfunc void launch_normal(T *a, const T mean, const T stddev, const unsigned int seed, const int size) { auto [numBlocks, blockSize] = BestDims(size); - kernel_normal<<>>(a,float(mean), float(stddev), seed, size); + kernel_normal<<>>(a, float(mean), float(stddev), seed, size); cudaError_t err = cudaGetLastError(); if (err != cudaSuccess) throw std::runtime_error("Failed to launch normal kernel"); diff --git a/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.cuh b/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.cuh index 1acbb322..ef8ffcbd 100644 --- a/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.cuh +++ b/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.cuh @@ -10,24 +10,36 @@ namespace deepx::tensorfunc { + //填充 template __global__ void kernel_constant(T *data, const T value, const int size); template void launch_constant(T *a, const T value, const int size); - + + //dropout + template + __global__ void dropout_kernel(T* A, const float p,const unsigned int seed,const int size); + + template + void launch_dropout(T* a, const float p,const unsigned int seed,const int size); + + //初始化 + //arange template __global__ void kernel_arange(T *data, const float start, const float step, const int size); template void launch_arange(T *a, const T start, const T step, const int size); - + + //uniform template __global__ void kernel_uniform(T *data, const float low, const float high, const unsigned int seed, const int size); template void launch_uniform(T *a, const T low, const T high, const unsigned int seed, const int size); + //normal template __global__ void kernel_normal(T *data, const float mean, const float stddev, const unsigned int seed, const int size); diff --git a/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.hpp b/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.hpp index 8340d2b6..b9078d89 100644 --- a/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.hpp +++ b/excuter/op-mem-cuda/src/deepx/tensorfunc/init_miaobyte.hpp @@ -19,6 +19,17 @@ namespace deepx::tensorfunc } }; + + + template + struct dropoutDispatcher + { + static void dropout(Tensor &A, const float p,const unsigned int seed) + { + launch_dropout(A.data, p, seed, A.shape.size); + } + }; + // arange template struct arangeDispatcher diff --git a/excuter/op-mem-cuda/src/deepx/tf/elementwise_basic.hpp b/excuter/op-mem-cuda/src/deepx/tf/elementwise_basic.hpp index 885bb5e6..e63cc74b 100644 --- a/excuter/op-mem-cuda/src/deepx/tf/elementwise_basic.hpp +++ b/excuter/op-mem-cuda/src/deepx/tf/elementwise_basic.hpp @@ -1034,73 +1034,7 @@ namespace deepx::tf } }; - // dropout - template - class Dropout : public TF - { - public: - Dropout(const vector &args, const vector &returns) - { - this->name = "dropout"; - this->metadata.author = Author::name(); - this->tftype = "elementwise"; - this->args = args; - this->returns = returns; - } - string math_formula() const override - { - return "T1.dropout(p,seed)->T3"; - } - shared_ptr clone() const override - { - return make_shared>(*this); - } - int run(shared_ptr mem, string &error) override - { - if (!checktensors({this->args[0].textvalue, this->returns[0].textvalue}, mem, error)) - { - return 1; - } - Precision a_type = mem->gettensor(this->args[0].textvalue).get()->shape.dtype; - Precision c_type = mem->gettensor(this->returns[0].textvalue).get()->shape.dtype; - if (a_type != c_type) - { - error = "Type mismatch: " + precision_str(a_type) + " != " + precision_str(c_type); - return 1; - } - switch (a_type) - { - case Precision::Float64: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1, mem), this->getvar(2, mem), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Float32: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1, mem), this->getvar(2, mem), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Float16: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1, mem), this->getvar(2, mem), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::BFloat16: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1, mem), this->getvar(2, mem), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Int64: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1, mem), this->getvar(2, mem), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Int32: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1, mem), this->getvar(2, mem), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Int16: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1, mem), this->getvar(2, mem), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Int8: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1, mem), this->getvar(2, mem), *mem->gettensor(this->returns[0].textvalue)); - break; - default: - error = "Unsupported dtype: " + precision_str(a_type); - return 1; - } - return 0; - } - }; + }; #endif // DEEPX_TF_ELEMENTWISE_BASIC_HPP diff --git a/excuter/op-mem-cuda/src/deepx/tf/init.hpp b/excuter/op-mem-cuda/src/deepx/tf/init.hpp index 055bb65c..b7a169db 100644 --- a/excuter/op-mem-cuda/src/deepx/tf/init.hpp +++ b/excuter/op-mem-cuda/src/deepx/tf/init.hpp @@ -10,7 +10,8 @@ #include "stdutil/num.hpp" namespace deepx::tf { - + // 填充 + // constant template class Constant : public TF { @@ -90,6 +91,71 @@ namespace deepx::tf }; }; + // dropout + template + class Dropout : public TF + { + public: + Dropout(const vector &args, const vector &returns) + { + this->name = "dropout"; + this->metadata.author = Author::name(); + this->tftype = "init"; + this->args = args; + this->returns = returns; + } + string math_formula() const override + { + return "dropout(p,seed)->A"; + } + shared_ptr clone() const override + { + return make_shared>(*this); + } + int run(shared_ptr mem, string &error) override + { + if (!checktensors({this->returns[0].textvalue }, mem, error)) + { + return 1; + } + Precision a_type = mem->gettensor(this->returns[0].textvalue).get()->shape.dtype; + + switch (a_type) + { + case Precision::Float64: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0, mem), this->getvar(1, mem)); + break; + case Precision::Float32: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0, mem), this->getvar(1, mem)); + break; + case Precision::Float16: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0, mem), this->getvar(1, mem)); + break; + case Precision::BFloat16: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0, mem), this->getvar(1, mem)); + break; + case Precision::Int64: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0, mem), this->getvar(1, mem)); + break; + case Precision::Int32: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0, mem), this->getvar(1, mem)); + break; + case Precision::Int16: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0, mem), this->getvar(1, mem)); + break; + case Precision::Int8: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0, mem), this->getvar(1, mem)); + break; + default: + error = "Unsupported dtype: " + precision_str(a_type); + return 1; + } + return 0; + } + }; + + // 初始化 + // arange template class Arange : public TF { @@ -173,7 +239,6 @@ namespace deepx::tf } return 0; } - }; template @@ -201,8 +266,8 @@ namespace deepx::tf string name = this->returns[0].textvalue; auto tensor = mem->gettensor(name).get(); auto type = tensor->shape.dtype; - int low_pos=0; - int high_pos=1; + int low_pos = 0; + int high_pos = 1; unsigned int seed = static_cast(this->getvar(2, mem)); switch (type) { @@ -262,7 +327,6 @@ namespace deepx::tf } return 0; } - }; // @@ -292,8 +356,8 @@ namespace deepx::tf string name = this->returns[0].textvalue; auto tensor = mem->gettensor(name).get(); auto type = tensor->shape.dtype; - int mean_pos=0; - int stddev_pos=1; + int mean_pos = 0; + int stddev_pos = 1; unsigned int seed = static_cast(this->getvar(2, mem)); switch (type) { diff --git a/excuter/op-mem-ompsimd/src/client/tfs.cpp b/excuter/op-mem-ompsimd/src/client/tfs.cpp index 32553db4..fcb2b1c8 100644 --- a/excuter/op-mem-ompsimd/src/client/tfs.cpp +++ b/excuter/op-mem-ompsimd/src/client/tfs.cpp @@ -59,28 +59,28 @@ namespace deepx::tf tffactory.add_tf(std::make_shared(vector( { Param("src", DataCategory::Tensor, Precision::Any), - }), + }), vector({ Param("dst", DataCategory::Tensor, Precision::Any), - + }))); // deltensor tffactory.add_tf(std::make_shared(vector( { - + }), vector({ - Param("t", DataCategory::Tensor, Precision::Any), + Param("t", DataCategory::Tensor, Precision::Any), }))); - //renametensor + // renametensor tffactory.add_tf(std::make_shared(vector( - { - - Param("new_name", DataCategory::Var, Precision::String), - }), - vector({ - Param("t", DataCategory::Tensor, Precision::Any), - }))); + { + + Param("new_name", DataCategory::Var, Precision::String), + }), + vector({ + Param("t", DataCategory::Tensor, Precision::Any), + }))); } // init @@ -89,16 +89,27 @@ namespace deepx::tf // constant author=miaobyte tffactory.add_tf(std::make_shared>(vector( { - + Param("value", DataCategory::Var, Precision::Any), }), vector({ - Param("t", DataCategory::Tensor, Precision::Any), + Param("t", DataCategory::Tensor, Precision::Any), }))); + // dropout author=miaobyte + tffactory.add_tf(std::make_shared>(vector( + { + + Param("p", DataCategory::Var, Precision::Float32), + Param("seed", DataCategory::Var, Precision::Int32), + }), + vector( + { + Param("A", DataCategory::Tensor, Precision::Any), + }))); // arange author=miaobyte tffactory.add_tf(std::make_shared>(vector( { - + Param("start", DataCategory::Var, Precision::Any), Param("step", DataCategory::Var, Precision::Any), }), @@ -108,27 +119,26 @@ namespace deepx::tf // uniform author=miaobyte tffactory.add_tf(std::make_shared>(vector( { - + Param("low", DataCategory::Var, Precision::Any), Param("high", DataCategory::Var, Precision::Any), Param("seed", DataCategory::Var, Precision::Int32), }), vector( - { + { Param("t", DataCategory::Tensor, Precision::Any), - } - ))); + }))); // normal author=miaobyte tffactory.add_tf(std::make_shared>(vector( { - + Param("mean", DataCategory::Var, Precision::Any), Param("std", DataCategory::Var, Precision::Any), Param("seed", DataCategory::Var, Precision::Int32), }), vector( { - Param("t", DataCategory::Tensor, Precision::Any), + Param("t", DataCategory::Tensor, Precision::Any), }))); } // io @@ -147,44 +157,43 @@ namespace deepx::tf Param("format", DataCategory::Var, Precision::String), }), vector())); - //save + // save opfactory.add_tf(std::make_shared(vector( - { - Param("t", DataCategory::Tensor, Precision::Any), - Param("path", DataCategory::Var, Precision::String), - }), - vector())); + { + Param("t", DataCategory::Tensor, Precision::Any), + Param("path", DataCategory::Var, Precision::String), + }), + vector())); - //load + // load opfactory.add_tf(std::make_shared(vector( - { - Param("path", DataCategory::Var, Precision::String), - }), - vector())); - //loadtensordata + { + Param("path", DataCategory::Var, Precision::String), + }), + vector())); + // loadtensordata opfactory.add_tf(std::make_shared(vector( - { - Param("path", DataCategory::Var, Precision::String), - }), - vector( - { - Param("t", DataCategory::Tensor, Precision::Any), - }))); + { + Param("path", DataCategory::Var, Precision::String), + }), + vector( + { + Param("t", DataCategory::Tensor, Precision::Any), + }))); } // elementwise void register_elementwise(TfFactory &tffactory) - { - // todtype + { + // todtype tffactory.add_tf(std::make_shared(vector( - { - Param("A", DataCategory::Tensor, Precision::Any), - }), - vector( - { - Param("C", DataCategory::Tensor, Precision::Any), - }))); - + { + Param("A", DataCategory::Tensor, Precision::Any), + }), + vector( + { + Param("C", DataCategory::Tensor, Precision::Any), + }))); // add author=miaobyte tffactory.add_tf(std::make_shared>(vector( @@ -465,17 +474,6 @@ namespace deepx::tf { Param("C", DataCategory::Tensor, Precision::Any), }))); - // dropout author=miaobyte - tffactory.add_tf(std::make_shared>(vector( - { - Param("A", DataCategory::Tensor, Precision::Any), - Param("p", DataCategory::Var, Precision::Float32), - Param("seed", DataCategory::Var, Precision::Int32), - }), - vector( - { - Param("C", DataCategory::Tensor, Precision::Any), - }) )); } // matmul void register_matmul(TfFactory &tffactory) @@ -546,15 +544,15 @@ namespace deepx::tf }))); // indexselect author=miaobyte tffactory.add_tf(std::make_shared>(vector( - { - Param("A", DataCategory::Tensor, Precision::Any), - Param("index", DataCategory::Tensor, Precision::Int32 | Precision::Int64), - Param("axis", DataCategory::Var, Precision::Int32), - }), - vector( - { - Param("B", DataCategory::Tensor, Precision::Any), - }))); + { + Param("A", DataCategory::Tensor, Precision::Any), + Param("index", DataCategory::Tensor, Precision::Int32 | Precision::Int64), + Param("axis", DataCategory::Var, Precision::Int32), + }), + vector( + { + Param("B", DataCategory::Tensor, Precision::Any), + }))); } // // reduce void register_reduce(TfFactory &tffactory) diff --git a/excuter/op-mem-ompsimd/src/deepx/tensorfunc/elementwise_miaobyte.hpp b/excuter/op-mem-ompsimd/src/deepx/tensorfunc/elementwise_miaobyte.hpp index b36902a4..de9e70a6 100644 --- a/excuter/op-mem-ompsimd/src/deepx/tensorfunc/elementwise_miaobyte.hpp +++ b/excuter/op-mem-ompsimd/src/deepx/tensorfunc/elementwise_miaobyte.hpp @@ -941,46 +941,6 @@ namespace deepx::tensorfunc } }; - // dropout - template - struct dropoutDispatcher - { - static void dropout(const Tensor &A, const float p, const unsigned int seed, Tensor &C) - { - if (A.shape == C.shape) - { - std::uniform_real_distribution distribution(0, 1); - std::default_random_engine generator; - if (seed != 0) - { - generator.seed(seed); - } - else - { - std::random_device rd; - generator.seed(rd()); - } - - A.shape.rangeElementwiseParallel([&A, &C, &p, &distribution, &generator](int i, int i_end) - { - for (int j = 0; j < i_end; j++) - { - double rand = distribution(generator); - if (rand < p) - { - C.data[i+j]=0; - } - else - { - C.data[i+j]=A.data[i+j]; - } - } }); - } - else - { - throw std::invalid_argument("shape mismatch"); - } - } - }; + }; #endif // DEEPX_OP_CPU_ELEMENTWISE_HPP \ No newline at end of file diff --git a/excuter/op-mem-ompsimd/src/deepx/tensorfunc/init_miaobyte.hpp b/excuter/op-mem-ompsimd/src/deepx/tensorfunc/init_miaobyte.hpp index 4afda582..ddde9c04 100644 --- a/excuter/op-mem-ompsimd/src/deepx/tensorfunc/init_miaobyte.hpp +++ b/excuter/op-mem-ompsimd/src/deepx/tensorfunc/init_miaobyte.hpp @@ -11,6 +11,7 @@ namespace deepx::tensorfunc { + //填充 //constant template struct constantDispatcher @@ -21,6 +22,40 @@ namespace deepx::tensorfunc } }; + // dropout + template + struct dropoutDispatcher + { + static void dropout(Tensor &A, const float p, const unsigned int seed) + { + + std::uniform_real_distribution distribution(0, 1); + std::default_random_engine generator; + if (seed != 0) + { + generator.seed(seed); + } + else + { + std::random_device rd; + generator.seed(rd()); + } + + A.shape.rangeElementwiseParallel([&A, &p, &distribution, &generator](int i, int i_end) + { + for (int j = 0; j < i_end; j++) + { + double rand = distribution(generator); + if (rand < p) + { + A.data[i+j]=0; + } + + } }); + + } + }; + //uniform template struct uniformDispatcher diff --git a/excuter/op-mem-ompsimd/src/deepx/tf/elementwise.hpp b/excuter/op-mem-ompsimd/src/deepx/tf/elementwise.hpp index 7c29ddab..bf579a7a 100644 --- a/excuter/op-mem-ompsimd/src/deepx/tf/elementwise.hpp +++ b/excuter/op-mem-ompsimd/src/deepx/tf/elementwise.hpp @@ -1901,63 +1901,6 @@ namespace deepx::tf } }; - //dropout - template - class Dropout : public TF - { - public: - Dropout(vector args, vector returns) - { - this->name = "dropout"; - this->metadata.author = Author::name(); - this->tftype = "elementwise"; - this->args = args; - this->returns = returns; - } - string math_formula() const override - { - return "A.dropout(p,seed)->C"; - } - shared_ptr clone() const override - { - return make_shared>(*this); - } - int run(shared_ptr mem, string &error) override - { - Precision a_type = mem->gettensor(this->args[0].textvalue).get()->shape.dtype; - Precision C_type = mem->gettensor(this->returns[0].textvalue).get()->shape.dtype; - if (a_type != C_type) - { - error = "Type mismatch: " + precision_str(a_type) + " != " + precision_str(C_type); - return 1; - } - switch (a_type) - { - case Precision::Float64: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1,mem,true), this->getvar(2,mem,true), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Float32: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1,mem,true), this->getvar(2,mem,true), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Int64: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1,mem,true), this->getvar(2,mem,true), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Int32: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1,mem,true), this->getvar(2,mem,true), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Int16: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1,mem,true), this->getvar(2,mem,true), *mem->gettensor(this->returns[0].textvalue)); - break; - case Precision::Int8: - tensorfunc::dropout(*mem->gettensor(this->args[0].textvalue), this->getvar(1,mem,true), this->getvar(2,mem,true), *mem->gettensor(this->returns[0].textvalue)); - break; - default: - error = "Unsupported dtype: " + precision_str(a_type); - return 1; - } - return 0; - } - }; - + }; #endif // DEEPX_TF_ELEMENTWISE_HPP diff --git a/excuter/op-mem-ompsimd/src/deepx/tf/init.hpp b/excuter/op-mem-ompsimd/src/deepx/tf/init.hpp index 756f880d..903c542a 100644 --- a/excuter/op-mem-ompsimd/src/deepx/tf/init.hpp +++ b/excuter/op-mem-ompsimd/src/deepx/tf/init.hpp @@ -6,7 +6,8 @@ #include "deepx/tensorfunc/init_miaobyte.hpp" #include "stdutil/num.hpp" namespace deepx::tf -{ +{ + //填充 // constant template class Constant : public TF @@ -87,6 +88,61 @@ namespace deepx::tf }; }; + //dropout + template + class Dropout : public TF + { + public: + Dropout(const vector &args, const vector &returns) + { + this->name = "dropout"; + this->metadata.author = Author::name(); + this->tftype = "init"; + this->args = args; + this->returns = returns; + } + string math_formula() const override + { + return "dropout(p,seed)->A"; + } + shared_ptr clone() const override + { + return make_shared>(*this); + } + int run(shared_ptr mem, string &error) override + { + Precision a_type = mem->gettensor(this->returns[0].textvalue).get()->shape.dtype; + + switch (a_type) + { + case Precision::Float64: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0,mem,true), this->getvar(1,mem,true)); + break; + case Precision::Float32: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0,mem,true), this->getvar(1,mem,true)); + break; + case Precision::Int64: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0,mem,true), this->getvar(1,mem,true)); + break; + case Precision::Int32: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0,mem,true), this->getvar(1,mem,true)); + break; + case Precision::Int16: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0,mem,true), this->getvar(1,mem,true)); + break; + case Precision::Int8: + tensorfunc::dropout(*mem->gettensor(this->returns[0].textvalue), this->getvar(0,mem,true), this->getvar(1,mem,true)); + break; + default: + error = "Unsupported dtype: " + precision_str(a_type); + return 1; + } + return 0; + } + }; + + + //初始化 // arange template class Arange : public TF diff --git a/front/py/deepx/nn/functional/elementwise.py b/front/py/deepx/nn/functional/elementwise.py index 76c22aca..5ab5d7b9 100644 --- a/front/py/deepx/nn/functional/elementwise.py +++ b/front/py/deepx/nn/functional/elementwise.py @@ -4,5 +4,4 @@ def rsqrt(input:Tensor)->Tensor: from .leaffunc_elementwise import sqrt return 1/sqrt(input) - - + \ No newline at end of file diff --git a/front/py/deepx/nn/functional/leaffunc_elementwise.py b/front/py/deepx/nn/functional/leaffunc_elementwise.py index 77bda9b2..6475105a 100644 --- a/front/py/deepx/nn/functional/leaffunc_elementwise.py +++ b/front/py/deepx/nn/functional/leaffunc_elementwise.py @@ -64,15 +64,3 @@ def todtype(t:Tensor,dest:Tensor): from .rtf_elementwise import rtf_todtype rtf_todtype(t,dest) -#dropout -def dropout(a:Tensor, p:float, out:Union[Tensor,str]='')->Tensor: - assert isinstance(a,Tensor) - outtensor=out - if isinstance(out,str) or out is None: - outtensor=newtensor(a.shape,dtype=a.dtype,name=out) - assert a.shape==outtensor.shape - - from .rtf_elementwise import rtf_dropout - rtf_dropout(a,p,outtensor,defaultauthor['dropout']) - return out - diff --git a/front/py/deepx/nn/functional/leaffunc_init.py b/front/py/deepx/nn/functional/leaffunc_init.py index 7711e2be..6ff30350 100644 --- a/front/py/deepx/nn/functional/leaffunc_init.py +++ b/front/py/deepx/nn/functional/leaffunc_init.py @@ -6,10 +6,7 @@ from deepx import Tensor,Number from .authormap import defaultauthor -# 命名规则 -# inplace操作的函数，其名为_后缀, 返回值为空 -# 非inplace操作的函数，其名为_后缀, 返回值为Tensor - +# 填充 def constant_(t:Tensor,value: Union[float,int])->Tensor: rtf_constant(t,value,defaultauthor['constant']) @@ -28,9 +25,19 @@ def ones(shape:tuple[int,...], dtype:str='float32',name:str=None)->Tensor: return constant(shape, value=1, dtype=dtype,name=name) def arange_(t:Tensor,start=0,step=1)->Tensor: - from .rtf_init import rtf_arange rtf_arange(t,start,step,defaultauthor['arange']) -#pytorch style + +def dropout(a:Tensor, p:float=0.5, seed:int=None)->Tensor: + assert isinstance(a,Tensor) + assert isinstance(p,float) and 0<=p<=1 + if seed is None: + import time,os + seed = int(time.time() * 1000) & 0xffffffff + seed = (seed + os.getpid()) & 0xffffffff + rtf_dropout(a,p,seed,defaultauthor['dropout']) + return a + +# 初始化 def arange(start:Number,end:Number,step:Number=1,dtype:str='float32',name:str=None)->Tensor: s =[int((end-start)/step)] outtensor=newtensor(s,dtype=dtype,name=name) @@ -41,7 +48,6 @@ def uniform_(t:Tensor,low=0, high=1,seed:int=None)->Tensor: if seed is None: seed = int(time.time() * 1000) & 0xffffffff seed = (seed + os.getpid()) & 0xffffffff - from .rtf_init import rtf_uniform rtf_uniform(t,low,high,seed,defaultauthor['uniform']) def uniform(shape:tuple[int,...],low=0, high=1,seed:int=None,dtype:str='float32',name:str=None)->Tensor: @@ -169,7 +175,6 @@ def normal_(t:Tensor,mean:float=0, stddev:float=1,seed:int=None)->Tensor: if seed is None: seed = int(time.time() * 1000) & 0xffffffff seed = (seed + os.getpid()) & 0xffffffff - from .rtf_init import rtf_normal rtf_normal(t,mean,stddev,seed,defaultauthor['normal']) def normal(shape:tuple[int,...],mean:float=0, stddev:float=1,seed:int=None,dtype:str='float32',name:str=None,author='miaobyte')->Tensor: diff --git a/front/py/deepx/nn/functional/rtf_elementwise.py b/front/py/deepx/nn/functional/rtf_elementwise.py index dc48ff46..4e20429d 100644 --- a/front/py/deepx/nn/functional/rtf_elementwise.py +++ b/front/py/deepx/nn/functional/rtf_elementwise.py @@ -3,7 +3,7 @@ from deepx.scheduler import send from .rtf import A_B_op_C,A_scalar_op_C,A_op_C - +# 四则运算 def rtf_add(a:Tensor, b:Tensor, out:Tensor, author='miaobyte')->Tensor: A_B_op_C("add",a,b,out,author) return out @@ -43,6 +43,7 @@ def rtf_rdivscalar(a:float, b:Tensor, out:Tensor, author='miaobyte')->Tensor: send(ir) return out +# 幂、指数 运算 def rtf_sqrt(a:Tensor, out:Tensor, author='miaobyte')->Tensor: A_op_C("sqrt",a,out,author) return out @@ -74,6 +75,7 @@ def rtf_rsqrt(a:Tensor, out:Tensor, author='miaobyte')->Tensor: A_op_C("rsqrt",a,out,author) return out +# 三角函数 def rtf_sin(a:Tensor, out:Tensor, author='miaobyte')->Tensor: A_op_C("sin",a,out,author) return out @@ -86,6 +88,7 @@ def rtf_tan(a:Tensor, out:Tensor, author='miaobyte')->Tensor: A_op_C("tan",a,out,author) return out +# 比较 def rtf_compare(a:Tensor, b:Tensor, out:Tensor, author='miaobyte')->Tensor: A_B_op_C("compare",a,b,out,author) return out @@ -110,6 +113,7 @@ def rtf_invert(a:Tensor, out:Tensor, author='miaobyte')->Tensor: A_op_C("invert",a,out,author) return out +# 类型转换 def rtf_todtype(t:Tensor,dest:Tensor): assert isinstance(t,Tensor) assert isinstance(dest,Tensor) @@ -120,6 +124,3 @@ def rtf_todtype(t:Tensor,dest:Tensor): ir=DeepxIR("todtype", args, returns,'') send(ir) -def rtf_dropout(a:Tensor, p:float, out:Tensor, author='miaobyte')->Tensor: - A_B_op_C("dropout",a,p,out,author) - return out \ No newline at end of file diff --git a/front/py/deepx/nn/functional/rtf_init.py b/front/py/deepx/nn/functional/rtf_init.py index 2fbcde8e..5da5a4b4 100644 --- a/front/py/deepx/nn/functional/rtf_init.py +++ b/front/py/deepx/nn/functional/rtf_init.py @@ -4,14 +4,7 @@ from typing import Union,Optional from .rtf import A_scalar_op - -def rtf_constant(t:Tensor,value:Union[float,int]=0,author='miaobyte')->Tensor: - args=[Param.varnum(value)] - returns=[Param.tensor(t)] - ir=DeepxIR("constant", args, returns,author) - send(ir) - return t - +# 初始化 def rtf_arange(t:Tensor,start:Optional[Union[float,int]]=0,step:Optional[Union[float,int]]=1,author='miaobyte')->Tensor: args=[Param.varnum(start),Param.varnum(step)] returns=[Param.tensor(t)] @@ -31,4 +24,22 @@ def rtf_normal(t:Tensor,mean:float=0, stddev:float=1,seed:int=0,author='miaobyte returns=[Param.tensor(t)] ir=DeepxIR("normal", args, returns,author) send(ir) - return t \ No newline at end of file + return t + +# 填充 +def rtf_constant(t:Tensor,value:Union[float,int]=0,author='miaobyte')->Tensor: + args=[Param.varnum(value)] + returns=[Param.tensor(t)] + ir=DeepxIR("constant", args, returns,author) + send(ir) + return t + +def rtf_dropout(a:Tensor, p:float, seed:int, author='miaobyte')->Tensor: + assert isinstance(p,float) and 0<=p<=1 + assert isinstance(seed,int) + + args=[Param.varnum(p),Param.varnum(seed)] + returns=[Param.tensor(a)] + ir=DeepxIR("dropout",args,returns,author) + send(ir) + return a \ No newline at end of file diff --git a/front/py/deepx/tensor/elementwise.py b/front/py/deepx/tensor/elementwise.py index 9cd78e62..16ceb1dd 100644 --- a/front/py/deepx/tensor/elementwise.py +++ b/front/py/deepx/tensor/elementwise.py @@ -164,7 +164,8 @@ def invert(self,out:Union[Tensor,str]='')->Tensor: @tensor_method -def dropout(self,p:float,out:Union[Tensor,str]=''): +def dropout_(self,p:float=0.5,seed:int=None): from deepx.nn.functional import dropout as dropout_func - return dropout_func(self,p,out) + dropout_func(self,p,seed) + return self diff --git a/front/py/deepx/transformer/models/llama/attention.py b/front/py/deepx/transformer/models/llama/attention.py index edb98f48..490501e4 100644 --- a/front/py/deepx/transformer/models/llama/attention.py +++ b/front/py/deepx/transformer/models/llama/attention.py @@ -1,6 +1,6 @@ from typing import Optional,Tuple from deepx.nn.modules import Module,Linear -from deepx import Tensor,repeat_kv,matmul,softmax,concat,arange +from deepx import Tensor,repeat_kv,matmul,softmax,concat,arange,dropout as dropout_func @@ -40,7 +40,7 @@ def eager_attention_forward( attn_weights = attn_weights + causal_mask attn_weights = softmax(attn_weights, dim=-1, dtype=query.dtype) - attn_weights = dropout(attn_weights, p=dropout, training=module.training) + attn_weights = dropout_func(attn_weights, p=dropout) attn_output = matmul(attn_weights, value_states) attn_output = attn_output.transpose(1, 2).contiguous() diff --git a/front/py/examples/3_functional/1_dropout.py b/front/py/examples/3_functional/1_dropout.py deleted file mode 100644 index 7aabaf7e..00000000 --- a/front/py/examples/3_functional/1_dropout.py +++ /dev/null @@ -1,9 +0,0 @@ -############-------PyTorch-------################ -print() - -import torch -import torch.nn.functional as F -torch_t = torch.empty(10, 10).uniform_(-1, 1) -torch_relu_t = F.dropout(torch_t) -print(torch_t) -print(torch_relu_t) diff --git a/front/py/examples/3_functional/elementwise_dropout.py b/front/py/examples/3_functional/elementwise_dropout.py new file mode 100644 index 00000000..0e7c0729 --- /dev/null +++ b/front/py/examples/3_functional/elementwise_dropout.py @@ -0,0 +1,18 @@ +############-------PyTorch-------################ +print() + +import torch +import torch.nn.functional as F +torch_t = torch.empty(10, 10).uniform_(-1, 1) +torch_dropout_t = F.dropout(torch_t) +print(torch_t) +print(torch_dropout_t) + + +############-------Deepx-------################ + +from deepx import uniform +t = uniform((10, 10), -1, 1) +dropout_t = t.clone() +dropout_t.dropout_(0.5) +dropout_t.print()