Skip to content

Commit 5fdfdaf

Browse files
miaobytemiaobyte
authored
remotefunc&leaffunc (#21)
* front:remotefunc * excuter:common/移除多余 * front:2种模式 1.eager不构建计算图，tensorname通过tensor_id(t)得到 2.各个框架，统一通过deepxIR，来构建计算图 * front:2种模式 1.eager不构建计算图，tensorname通过tensor_id(t)得到 2.各个框架，统一通过deepxIR，来构建计算图 * front:ir格式,init测试 * front:elementwise验证 * excuter&front:tensor.name启动支持 * excuter&front:tensor.name启动支持 * excuter&front:leaffunc_elementwise * excuter&front:leaffunc_changeshape * excuter&front:leaffunc_reduce全部完成测试 * excuter&front:leaffunc_reduce全部完成测试
1 parent 26f8638 commit 5fdfdaf

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+1767
-1641
lines changed

‎doc/excuter/op-mem-ompsimd/list.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
|-----------|--------|------------|--------------|----------------|
77
| reducemax | miaobyte | reducemax(tensor<any> A, vector<int32> axis, var<bool> keepdims)->(tensor<any> B) | B = reducemax(A, axis=[1 2], keepdims=false) | reducemax(tensor<any> A, vector<int32> axis, var<bool> keepdims)->(tensor<any> B) |
88
| broadcastTo | miaobyte | broadcastTo(tensor<any> A, vector<int32> new_shape)->(tensor<any> B) | T2 = T1.broadcastTo(new_shape=[4,3,2]) | broadcastTo(tensor<any> A, vector<int32> new_shape)->(tensor<any> B) |
9-
| concat | miaobyte | concat(listtensor<any> tensors, var<int32> axis)->(tensor<any> result) | Tresult = concat([T1, T2...], axis=3) | concat(listtensor<any> tensors, var<int32> axis)->(tensor<any> result) |
9+
| concat | miaobyte | concat(listtensor<any> tensors, var<int32> dim)->(tensor<any> result) | Tresult = concat([T1, T2...], axis=3) | concat(listtensor<any> tensors, var<int32> dim)->(tensor<any> result) |
1010
| transpose | miaobyte | transpose(tensor<any> A, vector<int32> dim_order)->(tensor<any> C) | T1.transpose(dimorder=[1,0])->T2 | transpose(tensor<any> A, vector<int32> dim_order)->(tensor<any> C) |
1111
| add | cblas | add(tensor<float64|float32> a, tensor<float64|float32> b)->(tensor<float64|float32> c) | T3=T1+T2 | add(tensor<float64|float32> a, tensor<float64|float32> b)->(tensor<float64|float32> c) |
1212
| add | miaobyte | add(tensor<any> a, tensor<any> b)->(tensor<any> c) | T3=T1+T2 | add(tensor<any> a, tensor<any> b)->(tensor<any> c) |

‎excuter/cpp-common/src/deepx/dtype.hpp‎

Lines changed: 98 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,14 @@
44
#include <string>
55
#include <cstdint>
66
#include <sstream>
7+
#include <vector>
8+
#include "stdutil/string.hpp"
9+
#include "stdutil/num.hpp"
710

811
namespace deepx
912
{
13+
using namespace std;
14+
1015
template <typename T>
1116
T to(const std::string &textvalue)
1217
{
@@ -134,7 +139,7 @@ namespace deepx
134139
Bool = 1 << 13, // 0010 0000 0000 0000
135140
String = 1 << 15, // 0100 0000 0000 0000
136141
// 常用组合
137-
Any = 0xFFFF, // 1111 1111 1111 1111
142+
Any = 0xFFFF, // 1111 1111 1111 1111
138143
Float = Float64 | Float32 | Float16 | BFloat16 | Float8E5M2 | Float8E4M3 | Float4E2M1,
139144
Float8 = Float8E5M2 | Float8E4M3, // 所有FP8格式
140145
Int = Int64 | Int32 | Int16 | Int8 | Int4
@@ -372,5 +377,97 @@ namespace deepx
372377
precision(precision_str));
373378
}
374379

380+
inline TypeDef autodtype(const std::string &param)
381+
{
382+
std::string type;
383+
std::string textvalue;
384+
std::vector<std::string> vectorvalues;
385+
bool vectorvalue = false;
386+
if (param.back() == ']')
387+
{
388+
size_t bracket_start = param.find('[');
389+
if (bracket_start != string::npos)
390+
{
391+
vectorvalue = true;
392+
// 提取方括号内的内容作为textvalue
393+
textvalue = param.substr(bracket_start + 1, param.length() - bracket_start - 2);
394+
// 提取方括号前的内容作为type
395+
type = param.substr(0, bracket_start);
396+
// 去除type两端的空格
397+
stdutil::trim(type);
398+
}
399+
}
400+
401+
if (!vectorvalue)
402+
{
403+
// 没有方括号，按空格分割
404+
stringstream ss(param);
405+
string first, second;
406+
ss >> first;
407+
if (ss >> second)
408+
{
409+
// 如果能读取到两个部分
410+
type = first;
411+
textvalue = second;
412+
}
413+
else
414+
{
415+
textvalue = first;
416+
}
417+
}
418+
// 处理向量值
419+
if (vectorvalue)
420+
{
421+
// 分割字符串为向量
422+
stringstream ss(textvalue);
423+
string item;
424+
while (getline(ss, item, ' '))
425+
{
426+
item.erase(0, item.find_first_not_of(" "));
427+
item.erase(item.find_last_not_of(" ") + 1);
428+
if (!item.empty())
429+
{
430+
vectorvalues.push_back(item);
431+
}
432+
}
433+
}
434+
435+
// 设置结果
436+
if (!type.empty())
437+
{
438+
return dtype(type);
439+
}
440+
else
441+
{
442+
// 没有显式类型声明,根据值推断
443+
if (vectorvalue)
444+
{
445+
if (!vectorvalues.empty())
446+
{
447+
if (is_integer(vectorvalues[0]))
448+
{
449+
return make_dtype(DataCategory::Vector, Precision::Int32);
450+
}
451+
else if (is_float(vectorvalues[0]))
452+
{
453+
return make_dtype(DataCategory::Vector, Precision::Float64);
454+
}
455+
else
456+
{
457+
return make_dtype(DataCategory::ListTensor, Precision::Any);
458+
}
459+
}
460+
else
461+
{
462+
return make_dtype(DataCategory::Vector, Precision::Any);
463+
}
464+
}
465+
else
466+
{
467+
return make_dtype(DataCategory::Var | DataCategory::Tensor, Precision::Any);
468+
}
469+
}
470+
}
471+
375472
} // namespace deepx
376473
#endif

‎excuter/cpp-common/src/deepx/tf/arg.hpp‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎excuter/cpp-common/src/deepx/tf/new.hpp‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

‎excuter/cpp-common/src/deepx/tf/tf.cpp‎

Lines changed: 44 additions & 115 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,41 @@
44

55
#include "deepx/tf/tf.hpp"
66
#include "stdutil/time.hpp"
7+
#include "stdutil/string.hpp"
78
namespace deepx::tf
8-
{
9+
{
10+
11+
12+
void Param::parse(const string &param)
13+
{
14+
//1. 按:分割类型和值
15+
size_t colon_pos = param.find(':');
16+
string type, textvalue;
17+
if (colon_pos != string::npos)
18+
{
19+
type = param.substr(0, colon_pos);
20+
stdutil::trimspace(type);
21+
textvalue = param.substr(colon_pos + 1);
22+
stdutil::trimspace(textvalue);
23+
}
24+
else
25+
{
26+
textvalue = param;
27+
stdutil::trimspace(textvalue);
28+
}
29+
if (!type.empty())
30+
{
31+
this->dtype = deepx::dtype(type);
32+
this->textvalue = textvalue;
33+
}
34+
else
35+
{
36+
this->dtype = deepx::dtype(textvalue);
37+
this->textvalue = textvalue;
38+
}
39+
40+
}
41+
// 分割主体和元数据
942
std::pair<string, string> split_body_metadata(const string &input)
1043
{
1144
size_t meta_pos = input.find("//");
@@ -30,11 +63,14 @@ namespace deepx::tf
3063
size_t space_pos = input_part.find(' ');
3164
size_t paren_pos = input_part.find('(');
3265
size_t name_end;
33-
34-
if (paren_pos != string::npos && (space_pos == string::npos || paren_pos < space_pos)) {
66+
67+
if (paren_pos != string::npos && (space_pos == string::npos || paren_pos < space_pos))
68+
{
3569
// 如果有括号且括号在空格之前,使用括号位置
3670
name_end = paren_pos;
37-
} else {
71+
}
72+
else
73+
{
3874
// 否则使用空格位置或字符串末尾
3975
name_end = space_pos != string::npos ? space_pos : input_part.length();
4076
}
@@ -150,112 +186,7 @@ namespace deepx::tf
150186
return value_str; // 默认作为字符串处理
151187
}
152188

153-
// 主参数解析函数,param已经去除两侧空格
154-
Param parse_param(const string &param)
155-
{
156-
Param result;
157-
if (param.empty())
158-
{
159-
return result;
160-
}
161-
162-
// 初始化变量
163-
string type;
164-
string textvalue;
165-
vector<string> vectorvalues;
166-
bool vectorvalue = false;
167-
168-
// 检查是否是向量值（以]结尾）
169-
if (param.back() == ']')
170-
{
171-
size_t bracket_start = param.find_last_of('[');
172-
if (bracket_start != string::npos)
173-
{
174-
vectorvalue = true;
175-
// 提取方括号内的内容作为textvalue
176-
textvalue = param.substr(bracket_start + 1, param.length() - bracket_start - 2);
177-
// 提取方括号前的内容作为type
178-
type = param.substr(0, bracket_start);
179-
// 去除type两端的空格
180-
type.erase(0, type.find_first_not_of(" "));
181-
type.erase(type.find_last_not_of(" ") + 1);
182-
}
183-
}
184-
185-
if (!vectorvalue)
186-
{
187-
// 没有方括号，按空格分割
188-
stringstream ss(param);
189-
string first, second;
190-
ss >> first;
191-
if (ss >> second)
192-
{
193-
// 如果能读取到两个部分
194-
type = first;
195-
textvalue = second;
196-
}
197-
else
198-
{
199-
textvalue = first;
200-
}
201-
}
202-
203-
// 处理向量值
204-
if (vectorvalue)
205-
{
206-
// 分割字符串为向量
207-
stringstream ss(textvalue);
208-
string item;
209-
while (getline(ss, item, ' '))
210-
{
211-
item.erase(0, item.find_first_not_of(" "));
212-
item.erase(item.find_last_not_of(" ") + 1);
213-
if (!item.empty())
214-
{
215-
vectorvalues.push_back(item);
216-
}
217-
}
218-
}
219-
220-
// 设置结果
221-
if (!type.empty())
222-
{
223-
result.dtype = dtype(type);
224-
}
225-
else
226-
{
227-
// 没有显式类型声明,根据值推断
228-
if (vectorvalue)
229-
{
230-
if (!vectorvalues.empty())
231-
{
232-
if (is_integer(vectorvalues[0])){
233-
result.dtype = make_dtype(DataCategory::Vector, Precision::Int32);
234-
}
235-
else if (is_float(vectorvalues[0]))
236-
{
237-
result.dtype = make_dtype(DataCategory::Vector, Precision::Float64);
238-
}
239-
else
240-
{
241-
result.dtype = make_dtype(DataCategory::ListTensor, Precision::Any);
242-
}
243-
}
244-
else
245-
{
246-
result.dtype = make_dtype(DataCategory::Vector, Precision::Any);
247-
}
248-
}
249-
else
250-
{
251-
result.dtype = make_dtype(DataCategory::Var|DataCategory::Tensor, Precision::Any);
252-
}
253-
}
254-
255-
result.textvalue = textvalue;
256-
return result;
257-
}
258-
189+
259190
// 解析参数列表
260191
vector<Param> parse_params(const string &params_str)
261192
{
@@ -266,14 +197,14 @@ namespace deepx::tf
266197
while (getline(params_ss, param, ','))
267198
{
268199
// 去除首尾空格
269-
param.erase(0, param.find_first_not_of(" "));
270-
param.erase(param.find_last_not_of(" ") + 1);
200+
stdutil::trimspace(param);
271201

272202
if (param.empty())
273203
continue;
274204

275205
// 解析单个参数
276-
Param parsed_param = parse_param(param);
206+
Param parsed_param;
207+
parsed_param.parse(param);
277208
params.push_back(parsed_param);
278209
}
279210

@@ -452,6 +383,4 @@ namespace deepx::tf
452383
return true;
453384
}
454385

455-
456-
457386
}

0 commit comments

Comments
 (0)