TorchNN/ReadDataset.lua at master · Legomaniak/TorchNN · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
require 'TensorSaveLoad'
require 'gnuplot'
function ReadDataset(fname)
  local DoPCA = false

  --local file = NactiData(settings.listFolder .. fname..'BatchIn');
  local file = NactiData(settings.listFolder .. fname);
  local input = file--:float();

  --file = NactiData(settings.listFolder .. fname..'BatchOut');
  file = NactiData(settings.listFolder .. fname..'_OUT');
  local output = file--:float();

  local min = torch.min(input);
  input = input - min;

  local max = torch.max(input);
  input = input / max;

  if (DoPCA) then
    local wavelength = torch.load(settings.listFolder .. 'DataMixedPCA');
    local data = torch.Tensor(dataset.nSamplesList:size(1),dataset.nSamplesList:size(2),wavelength:size(1))

    for v = 1, wavelength:size(1), 1 do
     data[{{},{},{v}}] = input:select(3,wavelength[v][1])
    end
    input = data;
  end
    input = input:transpose(1,3);
    input = input:transpose(2,3);
  return input,output;
end

function ReadDataAllOld(list)
  local t ={}
  local pocet = table.getn(list)
  table.insert(t,pocet)
  local file = NactiData(Settings.ListFolder .. list[1]);
  for i = 1,file:dim() do
    table.insert(t,file:size(i))
  end
  local data = torch.Tensor(torch.LongStorage(t))
  data[1] = file
  for i = 2,pocet do
    file = NactiData(Settings.ListFolder .. list[i]);
    data[i] = file
  end
  return data
end

--function ReadDataAll(list)
--  local t ={}
--  local pocet = table.getn(list)
--  table.insert(t,pocet)
--  local file = NactiData(list[1]);
--  for i = 1,file:dim() do
--    table.insert(t,file:size(i))
--  end
--  local data = torch.Tensor(torch.LongStorage(t))
--  data[1] = file
--  for i = 2,pocet do
--    file = NactiData(list[i]);
--    data[i] = file
--  end
--  return data
--end
function ReadDataAll(list)
  local data ={}
  local pocet = table.getn(list)
  for i = 1,pocet do
   local file = NactiData(list[i]);
   table.insert(data,file)
  end
  return data
end
--function ReadDir(dir)
--Lists = {}
--listIdent = 'norm'
--ListsOut = {}
--listOutIdent = 'OUT'

--for file in lfs.dir(dir) do
--    if lfs.attributes(dir..file,"mode") == "file" then
--      if string.find(file,listIdent) then
--        table.insert(Lists,dir..file);
--      elseif string.find(file,listOutIdent) then
--        table.insert(ListsOut,dir..file);
--      end
--    end
--end
--return Lists,ListsOut
--end

function ReadDir(dir,listIdent,listOutIdent)
Lists = {}
ListsOut = {}

for file in lfs.dir(dir) do
    if lfs.attributes(dir..file,"mode") == "file" then
      if string.find(file,listIdent) then
        table.insert(Lists,dir..file);
      elseif string.find(file,listOutIdent) then
        table.insert(ListsOut,dir..file);
      end
    end
end
return Lists,ListsOut
end