Line data Source code
1 : // SPDX-License-Identifier: Apache-2.0
2 : /**
3 : * Copyright (C) 2023 Jijoong Moon <jijoong.moon@samsung.com>
4 : *
5 : * @file dir_data_producers.h
6 : * @date 24 Feb 2023
7 : * @brief This file contains dir data producers, reading from the files in
8 : * directory
9 : * @see https://github.com/nnstreamer/nntrainer
10 : * @author Jijoong Moon <jijoong.moon@samsung.com>
11 : * @bug No known bugs except for NYI items
12 : *
13 : */
14 :
15 : #include <dir_data_producers.h>
16 : #include <filesystem>
17 :
18 : #include <memory>
19 : #include <numeric>
20 : #include <random>
21 : #include <sys/stat.h>
22 : #include <vector>
23 :
24 : #include <common_properties.h>
25 : #include <nntrainer_error.h>
26 : #include <node_exporter.h>
27 : #include <util_func.h>
28 :
29 : /**
30 : * @brief this function helps to read the image
31 : * currently only bmp image is supported and extend the image type will be
32 : * remain as TODO. ( BGR --> RGB )
33 : * @param path file path
34 : * @param inputs float data for image pixel
35 : * @param width width
36 : * @param height height
37 : */
38 0 : static void readImage(const std::string path, float *input, unsigned int width,
39 : unsigned int height) {
40 0 : FILE *f = fopen(path.c_str(), "rb");
41 :
42 0 : if (f == nullptr)
43 0 : throw std::invalid_argument("Cannot open file: " + path);
44 :
45 : unsigned char info[54];
46 : size_t result = fread(info, sizeof(unsigned char), 54, f);
47 0 : NNTR_THROW_IF(result != 54, std::invalid_argument)
48 : << "Cannot read bmp header";
49 :
50 0 : size_t row_padded = (width * 3 + 3) & (~3);
51 0 : unsigned char *data = new unsigned char[row_padded];
52 :
53 0 : for (unsigned int i = 0; i < height; i++) {
54 : result = fread(data, sizeof(unsigned char), row_padded, f);
55 0 : NNTR_THROW_IF(result != row_padded, std::invalid_argument)
56 : << "Cannot read bmp pixel data";
57 :
58 0 : for (unsigned int j = 0; j < width; j++) {
59 :
60 0 : input[height * i + j] = (float)data[j * 3 + 2];
61 :
62 0 : input[(height * width) + height * i + j] = (float)data[j * 3 + 1];
63 :
64 0 : input[(height * width) * 2 + height * i + j] = (float)data[j * 3];
65 : }
66 : }
67 :
68 0 : delete[] data;
69 0 : fclose(f);
70 0 : }
71 :
72 : namespace nntrainer {
73 :
74 0 : DirDataProducer::DirDataProducer() :
75 0 : dir_data_props(new Props()), num_class(0), num_data_total(0) {}
76 :
77 0 : DirDataProducer::DirDataProducer(const std::string &dir_path) :
78 0 : dir_data_props(new Props(props::DirPath(dir_path))),
79 0 : num_class(0),
80 0 : num_data_total(0) {}
81 :
82 0 : DirDataProducer::~DirDataProducer() {}
83 :
84 0 : const std::string DirDataProducer::getType() const {
85 0 : return DirDataProducer::type;
86 : }
87 :
88 0 : bool DirDataProducer::isMultiThreadSafe() const {
89 : /// @todo make this true, it is needed to test multiple worker scenario
90 0 : return false;
91 : }
92 :
93 0 : void DirDataProducer::setProperty(const std::vector<std::string> &properties) {
94 0 : auto left = loadProperties(properties, *dir_data_props);
95 0 : NNTR_THROW_IF(!left.empty(), std::invalid_argument)
96 : << "There are unparsed properties, size: " << left.size();
97 0 : }
98 :
99 : DataProducer::Generator
100 0 : DirDataProducer::finalize(const std::vector<TensorDim> &input_dims,
101 : const std::vector<TensorDim> &label_dims,
102 : void *user_data) {
103 :
104 0 : const auto &dir_path = std::get<props::DirPath>(*dir_data_props).get();
105 :
106 0 : for (const auto &entry : std::filesystem::directory_iterator(dir_path))
107 0 : class_names.push_back(entry.path().string());
108 :
109 0 : num_class = class_names.size();
110 :
111 : size_t id = 0;
112 : size_t num_data = 0;
113 0 : for (auto &c_name : class_names) {
114 : num_data = 0;
115 0 : std::filesystem::directory_iterator itr(c_name);
116 0 : while (itr != std::filesystem::end(itr)) {
117 0 : const std::filesystem::directory_entry &entry = *itr;
118 0 : std::string p = std::filesystem::absolute(entry.path()).string();
119 0 : if (p.compare(".") && p.compare("..")) {
120 0 : num_data++;
121 0 : data_list.push_back(std::make_pair((unsigned int)id, p));
122 : }
123 0 : itr++;
124 : }
125 :
126 0 : id++;
127 0 : num_data_total += num_data;
128 : }
129 :
130 : /// @todo expand this to non onehot case
131 0 : NNTR_THROW_IF(std::any_of(label_dims.begin(), label_dims.end(),
132 : [](const TensorDim &dim) {
133 : return dim.channel() != 1 || dim.height() != 1;
134 : }),
135 : std::invalid_argument)
136 : << "Label dimension containing channel or height not allowed";
137 :
138 0 : auto sz = size(input_dims, label_dims);
139 :
140 0 : NNTR_THROW_IF(sz == 0, std::invalid_argument)
141 : << "size is zero, data producer does not provide anything";
142 :
143 0 : return [sz, input_dims, this](unsigned int idx, std::vector<Tensor> &inputs,
144 : std::vector<Tensor> &labels) {
145 0 : NNTR_THROW_IF(idx >= sz, std::range_error)
146 : << "given index is out of bound, index: " << idx << " size: " << sz;
147 :
148 0 : std::string file_name = data_list[idx].second;
149 :
150 0 : readImage(file_name, inputs[0].getData(), input_dims[0].width(),
151 0 : input_dims[0].height());
152 :
153 0 : unsigned int c_id = data_list[idx].first;
154 :
155 0 : std::memset(labels[0].getData(), 0, num_class * sizeof(float));
156 :
157 0 : labels[0].getData()[c_id] = 1.0;
158 :
159 0 : return idx == sz - 1;
160 0 : };
161 : }
162 :
163 : unsigned int
164 0 : DirDataProducer::size(const std::vector<TensorDim> &input_dims,
165 : const std::vector<TensorDim> &label_dims) const {
166 :
167 0 : return num_data_total;
168 : }
169 :
170 : } // namespace nntrainer
|
