Line data Source code
1 : // SPDX-License-Identifier: Apache-2.0
2 : /**
3 : * Copyright (C) 2021 Parichay Kapoor <pk.kapoor@samsung.com>
4 : *
5 : * @file layer_node.cpp
6 : * @date 1 April 2021
7 : * @see https://github.com/nnstreamer/nntrainer
8 : * @author Parichay Kapoor <pk.kapoor@samsung.com>
9 : * @author Debadri Samaddar <s.debadri@samsung.com>
10 : * @bug No known bugs except for NYI items
11 : * @brief This is the layer node for network graph
12 : */
13 :
14 : #include "layer_context.h"
15 : #include <algorithm>
16 : #include <cmath>
17 : #include <iterator>
18 : #include <stdexcept>
19 : #include <utility>
20 :
21 : #include <activation_layer.h>
22 : #include <base_properties.h>
23 : #include <bn_layer.h>
24 : #include <common_properties.h>
25 : #include <connection.h>
26 : #include <context.h>
27 : #include <engine.h>
28 : #include <layer_node.h>
29 : #include <nntrainer_error.h>
30 : #include <nntrainer_log.h>
31 : #include <node_exporter.h>
32 : #include <profiler.h>
33 : #include <time_dist.h>
34 : #include <tracer.h>
35 : #include <util_func.h>
36 :
37 : #ifdef ENABLE_OPENCL
38 : #include <cl_context.h>
39 : #endif
40 :
41 : namespace nntrainer {
42 :
43 : #ifdef PROFILE
44 : static constexpr const char *FORWARD_SUFFIX = ":forward";
45 : static constexpr const char *CALC_DERIV_SUFFIX = ":calcDeriv";
46 : static constexpr const char *CALC_GRAD_SUFFIX = ":calcGrad";
47 : #endif
48 :
49 : namespace props {
50 :
51 : /**
52 : * @brief Flatten property, true if needs flatten layer afterwards
53 : */
54 6191 : class Flatten : public Property<bool> {
55 : public:
56 : Flatten() : Property<bool>() {} /**< has default value of 0 */
57 : static constexpr const char *key = "flatten"; /**< unique key to access */
58 : using prop_tag = bool_prop_tag; /**< property type */
59 : };
60 :
61 : /**
62 : * @brief Distribute property, true if it distribute across layer
63 : *
64 : */
65 6191 : class Distribute : public Property<bool> {
66 : public:
67 6191 : Distribute() : Property<bool>() {}
68 : static constexpr const char *key = "distribute";
69 : using prop_tag = bool_prop_tag;
70 : };
71 :
72 : /**
73 : * @brief Loss property, this defines loss specification of layer
74 : *
75 : */
76 : class Loss : public Property<float> {
77 :
78 : public:
79 : /**
80 : * @brief Construct a new loss object with a default value 0.0
81 : *
82 : */
83 6191 : Loss(float value = 0.0) : nntrainer::Property<float>(value) {}
84 : static constexpr const char *key = "loss"; /**< unique key to access */
85 : using prop_tag = float_prop_tag; /**< property type */
86 :
87 : /**
88 : * @brief LossSpec validator
89 : * @todo detect when loss becomes Nan is useful. But it will need dedicated
90 : * throw
91 : * @param v float to validate
92 : * @retval true if is valid number
93 : * @retval false if it is nan
94 : */
95 34147 : bool isValid(const float &v) const override {
96 34147 : if (std::isnan(v)) {
97 10 : ml_logw("loss value is NAN");
98 : }
99 :
100 34147 : return true;
101 : }
102 : };
103 :
104 : /**
105 : * @brief Input shape property which saves a single tensor shape
106 : * (practically, std::array<InputShape> is used)
107 : *
108 : */
109 5075 : class InputShape : public GenericShape {
110 :
111 : public:
112 : static constexpr const char *key = "input_shape"; /**< unique key to access */
113 : using prop_tag = dimension_prop_tag; /**< property type */
114 : };
115 :
116 : /**
117 : * @brief properties for shared from
118 : *
119 : */
120 12382 : class SharedFrom : public Name {
121 : public:
122 : static constexpr const char *key = "shared_from"; /**< unique key to access */
123 : using prop_tag = str_prop_tag; /**< property type */
124 : };
125 :
126 : } // namespace props
127 :
128 : /**
129 : * @brief Destroy the Layer Node object
130 : *
131 : */
132 24763 : LayerNode::~LayerNode() = default;
133 :
134 : /**
135 : * @brief get the compute engine property from property string vector
136 : * : default is CPU
137 : * @return LayerComputeEngine Enum : CPU, GPU, QNN
138 : *
139 : */
140 : ml::train::LayerComputeEngine
141 0 : getComputeEngine(const std::vector<std::string> &props) {
142 0 : for (auto &prop : props) {
143 : std::string key, value;
144 0 : int status = nntrainer::getKeyValue(prop, key, value);
145 0 : if (nntrainer::istrequal(key, "engine")) {
146 : constexpr const auto data =
147 : std::data(props::ComputeEngineTypeInfo::EnumList);
148 0 : for (unsigned int i = 0;
149 0 : i < props::ComputeEngineTypeInfo::EnumList.size(); ++i) {
150 0 : if (nntrainer::istrequal(value.c_str(),
151 0 : props::ComputeEngineTypeInfo::EnumStr[i])) {
152 0 : return data[i];
153 : }
154 : }
155 : }
156 : }
157 :
158 : return ml::train::LayerComputeEngine::CPU;
159 : }
160 :
161 : /**
162 : * @brief Layer factory creator with constructor
163 : */
164 : std::unique_ptr<LayerNode>
165 91 : createLayerNode(const ml::train::LayerType &type,
166 : const std::vector<std::string> &properties) {
167 91 : auto &eg = nntrainer::Engine::Global();
168 180 : return createLayerNode(eg.createLayerObject(type, properties), properties);
169 : }
170 :
171 : /**
172 : * @brief Layer factory creator with constructor
173 : */
174 : std::unique_ptr<LayerNode>
175 1771 : createLayerNode(const std::string &type,
176 : const std::vector<std::string> &properties) {
177 1771 : auto &eg = nntrainer::Engine::Global();
178 3534 : return createLayerNode(eg.createLayerObject(type, properties), properties);
179 : }
180 :
181 : /**
182 : * @brief Layer factory creator with constructor
183 : */
184 : std::unique_ptr<LayerNode>
185 6190 : createLayerNode(std::unique_ptr<nntrainer::Layer> &&layer,
186 : const std::vector<std::string> &properties) {
187 6190 : auto lnode = std::make_unique<LayerNode>(std::move(layer));
188 :
189 6190 : lnode->setProperty(properties);
190 :
191 6188 : return lnode;
192 : }
193 :
194 6191 : LayerNode::LayerNode(std::unique_ptr<nntrainer::Layer> &&l) :
195 : layer(std::move(l)),
196 6191 : inplace_type(InPlaceType::NONE),
197 6191 : needs_calc_derivative(false),
198 6191 : needs_calc_gradient(false),
199 :
200 : output_connections(),
201 : run_context(nullptr),
202 : layer_node_props(new PropsType(
203 12382 : props::Name(), props::Distribute(), props::Trainable(), {}, {},
204 18573 : props::SharedFrom(), props::ClipGradByGlobalNorm(), props::Packed(),
205 18573 : props::WeightDtype(), props::LossScaleForMixed(), props::ComputeEngine())),
206 : layer_node_props_realization(
207 6191 : new RealizationPropsType(props::Flatten(), props::Activation())),
208 12382 : loss(new props::Loss()),
209 : exec_order({0, 0, 0, 0}),
210 6191 : needs_restore_data(false),
211 12382 : data_type({TensorDim::DataType::FP32, TensorDim::DataType::FP32}) {
212 12382 : if (layer && layer->getType() == TimeDistLayer::type) {
213 0 : std::get<props::Distribute>(*layer_node_props).set(true);
214 : }
215 6191 : }
216 :
217 23226 : void LayerNode::setProperty(const std::vector<std::string> &properties) {
218 23226 : auto left_properties = loadProperties(properties, *layer_node_props);
219 : left_properties =
220 23217 : loadProperties(left_properties, *layer_node_props_realization);
221 23215 : layer->setProperty(left_properties);
222 :
223 23208 : if (getType() == ActivationLayer::type) {
224 : auto &act_prop = std::get<props::Activation>(*layer_node_props_realization);
225 2692 : if (!act_prop.empty()) {
226 10536 : layer->setProperty({"activation=" + to_string(act_prop)});
227 : }
228 : }
229 23217 : }
230 :
231 0 : std::string LayerNode::getProperty(const std::string &key) {
232 0 : if (layer_node_props) {
233 : std::string result = find_in_tuple(*layer_node_props, key);
234 0 : if (!result.empty()) {
235 0 : return result;
236 : }
237 : }
238 :
239 0 : if (layer_node_props_realization) {
240 : std::string result = find_in_tuple(*layer_node_props_realization, key);
241 0 : if (!result.empty()) {
242 0 : return result;
243 : }
244 : }
245 :
246 0 : return layer->getProperty(key);
247 : }
248 :
249 4 : void LayerNode::setWeights(const std::vector<float *> weights) {
250 5 : NNTR_THROW_IF(!run_context, std::runtime_error)
251 : << __func__ << " layer needs to be finalized first!";
252 :
253 4 : NNTR_THROW_IF(getNumWeights() != weights.size(), std::runtime_error)
254 : << __func__ << " Number of Weights dismatch!";
255 :
256 : // Needs Deep copy
257 6 : for (unsigned int idx = 0; idx < getNumWeights(); ++idx) {
258 4 : Tensor &w = getWeight(idx);
259 4 : std::copy(weights[idx], weights[idx] + w.size(), w.getData());
260 : }
261 2 : }
262 :
263 8603 : const unsigned LayerNode::getInputConnectionIndex(unsigned nth) const {
264 : auto &input_conns =
265 : std::get<std::vector<props::InputConnection>>(*layer_node_props);
266 8603 : return input_conns.at(nth).get().getIndex();
267 : }
268 :
269 14103 : const std::string &LayerNode::getInputConnectionName(unsigned nth) const {
270 : auto &input_conns =
271 : std::get<std::vector<props::InputConnection>>(*layer_node_props);
272 14103 : return input_conns.at(nth).get().getName();
273 : }
274 :
275 391 : void LayerNode::setInputConnectionIndex(unsigned nth, unsigned index) {
276 : auto &input_conns =
277 : std::get<std::vector<props::InputConnection>>(*layer_node_props);
278 391 : input_conns.at(nth).get().getIndex() = index;
279 391 : }
280 :
281 397 : void LayerNode::setInputConnectionName(unsigned nth, const std::string &name) {
282 : auto &input_conns =
283 : std::get<std::vector<props::InputConnection>>(*layer_node_props);
284 397 : input_conns.at(nth).get().getName() = name;
285 397 : }
286 :
287 7786 : const Connection *LayerNode::getOutputConnection(unsigned nth) const {
288 7786 : return output_connections.at(nth).get();
289 : }
290 :
291 4519 : void LayerNode::setOutputConnection(unsigned nth, const std::string &name,
292 : unsigned index) {
293 4519 : if (nth >= output_connections.size()) {
294 4505 : output_connections.resize(nth + 1);
295 : }
296 :
297 : auto &con = output_connections[nth];
298 : // Should be override connection for the batch normalization realizer
299 : // NNTR_THROW_IF(con, std::invalid_argument)
300 : // << "cannot override connection, this slot is reserved for "
301 : // << con->toString();
302 :
303 9038 : con = std::make_unique<Connection>(name, index);
304 4519 : }
305 :
306 0 : void LayerNode::setComputeEngine(
307 : const ml::train::LayerComputeEngine &compute_engine) {
308 : // setting compute_engine of LayerNode
309 : // can be reused later to propagate this info
310 0 : this->compute_engine = compute_engine;
311 0 : }
312 :
313 209940 : const std::string LayerNode::getName() const {
314 : auto &name = std::get<props::Name>(*layer_node_props);
315 209940 : return name.empty() ? "" : name.get();
316 : }
317 :
318 16 : std::ostream &operator<<(std::ostream &out, const LayerNode &l) {
319 :
320 : auto &input_connections =
321 : std::get<std::vector<props::InputConnection>>(*l.layer_node_props);
322 :
323 64 : out << "[" << l.getName() << '/' << l.getType() << "]\n";
324 16 : auto print_vector = [&out](const auto &cons, const std::string &title) {
325 32 : out << title << "[" << cons.size() << "] ";
326 16 : for (auto &con : cons) {
327 0 : out << con.toString() << ' ';
328 : }
329 16 : out << '\n';
330 16 : };
331 :
332 16 : auto print_vector_2 = [&out](const auto &cons, const std::string &title) {
333 32 : out << title << "[" << cons.size() << "] ";
334 16 : for (auto &con : cons) {
335 0 : out << con->toString() << ' ';
336 : }
337 16 : out << '\n';
338 16 : };
339 :
340 32 : print_vector(
341 32 : std::vector<Connection>(input_connections.begin(), input_connections.end()),
342 : " input_connections");
343 16 : print_vector_2(l.output_connections, "output_connections");
344 16 : return out;
345 : }
346 :
347 3949 : ActivationType LayerNode::getActivationType() const {
348 : auto &act_prop = std::get<props::Activation>(*layer_node_props_realization);
349 3949 : if (act_prop.empty()) {
350 : return ActivationType::ACT_NONE;
351 : }
352 :
353 482 : return act_prop;
354 : }
355 :
356 27685 : unsigned int LayerNode::getNumInputConnections() const {
357 : auto &input_conns =
358 : std::get<std::vector<props::InputConnection>>(*layer_node_props);
359 27685 : return input_conns.size();
360 : }
361 :
362 8659 : unsigned int LayerNode::getNumOutputConnections() const {
363 8659 : return output_connections.size();
364 : }
365 :
366 19254 : const std::vector<std::string> LayerNode::getInputLayers() const {
367 : auto &input_connections =
368 : std::get<std::vector<props::InputConnection>>(*layer_node_props);
369 : std::vector<std::string> names;
370 19254 : names.reserve(input_connections.size());
371 19254 : std::transform(
372 : input_connections.begin(), input_connections.end(),
373 : std::back_inserter(names),
374 : [](const Connection &con) -> const auto & { return con.getName(); });
375 19254 : return names;
376 0 : }
377 :
378 9494 : const std::vector<std::string> LayerNode::getOutputLayers() const {
379 : std::vector<std::string> names;
380 9494 : names.reserve(output_connections.size());
381 :
382 18650 : for (auto &conn : output_connections) {
383 9156 : if (conn == nullptr) {
384 32 : ml_logw("intermediate output is empty for layer: %s", getName().c_str());
385 16 : continue;
386 : }
387 9140 : names.push_back(conn->getName());
388 : }
389 9494 : return names;
390 0 : }
391 :
392 3742 : ActivationType LayerNode::getActivationToBeRealized() const {
393 3742 : if (getType() == ActivationLayer::type)
394 : return ActivationType::ACT_NONE;
395 : else
396 3742 : return getActivationType();
397 : }
398 :
399 117652 : const std::string LayerNode::getType() const { return getLayer()->getType(); }
400 :
401 87746 : bool LayerNode::getTrainable() const {
402 87746 : if (run_context)
403 : /**
404 : * if a layer does not contain any weights, it will be treated as a
405 : * non-trainable layer.
406 : */
407 110356 : return std::get<props::Trainable>(*layer_node_props) &&
408 110356 : (run_context->getNumWeights() > 0);
409 : else
410 32253 : return std::get<props::Trainable>(*layer_node_props);
411 : }
412 :
413 4060 : bool LayerNode::getFlatten() const {
414 : auto &flatten = std::get<props::Flatten>(*layer_node_props_realization);
415 4060 : if (flatten.empty()) {
416 : return false;
417 : }
418 2 : return flatten.get();
419 : }
420 :
421 9347 : std::string LayerNode::getSharedFrom() const {
422 : auto &shared_from = std::get<props::SharedFrom>(*layer_node_props);
423 9347 : return shared_from.empty() ? "" : shared_from.get();
424 : }
425 :
426 79308 : bool LayerNode::getDistribute() const {
427 : auto &distribute = std::get<props::Distribute>(*layer_node_props);
428 79308 : if (distribute.empty()) {
429 : return false;
430 : }
431 11 : return distribute.get();
432 : }
433 :
434 177637 : const nntrainer::Layer *LayerNode::getLayer() const {
435 177637 : if (run_context && getDistribute())
436 5 : return static_cast<TimeDistLayer *>(layer.get())->getDistLayer();
437 : else
438 177632 : return layer.get();
439 : }
440 :
441 3066 : nntrainer::Layer *LayerNode::getLayer() {
442 3066 : if (run_context && getDistribute())
443 2 : return static_cast<TimeDistLayer *>(layer.get())->getDistLayer();
444 : else
445 3064 : return layer.get();
446 : }
447 :
448 459 : void LayerNode::setOutputLayers(const std::vector<std::string> &layers) {
449 : output_connections.clear();
450 459 : output_connections.reserve(layers.size());
451 459 : std::transform(
452 : layers.begin(), layers.end(), std::back_inserter(output_connections),
453 458 : [](const std::string &id) { return std::make_unique<Connection>(id); });
454 459 : }
455 :
456 7172 : bool LayerNode::hasInputShapeProperty() const {
457 : auto &input_shapes =
458 : std::get<std::vector<props::InputShape>>(*layer_node_props);
459 :
460 7172 : return !input_shapes.empty() &&
461 : std::all_of(input_shapes.begin(), input_shapes.end(),
462 7172 : [](const auto &input) { return !input.empty(); });
463 : }
464 :
465 2882 : const std::vector<TensorDim> LayerNode::getInputDimensions() const {
466 2883 : NNTR_THROW_IF(!run_context, std::runtime_error)
467 : << __func__ << " layer needs to be finalized first!";
468 2881 : auto sz = run_context->getNumInputs();
469 : std::vector<TensorDim> dims;
470 2881 : dims.reserve(sz);
471 :
472 6320 : for (auto i = 0u; i < sz; ++i) {
473 6878 : dims.push_back(run_context->getInput(i).getDim());
474 : }
475 :
476 2881 : return dims;
477 0 : }
478 :
479 2586 : const std::vector<TensorDim> LayerNode::getOutputDimensions() const {
480 2587 : NNTR_THROW_IF(!run_context, std::runtime_error)
481 : << __func__ << " layer needs to be finalized first!";
482 2585 : auto sz = run_context->getNumOutputs();
483 : std::vector<TensorDim> dims;
484 2585 : dims.reserve(sz);
485 :
486 5626 : for (auto i = 0u; i < sz; ++i) {
487 6082 : dims.push_back(run_context->getOutput(i).getDim());
488 : }
489 :
490 2585 : return dims;
491 0 : }
492 :
493 2542 : void LayerNode::exportTo(Exporter &exporter,
494 : const ml::train::ExportMethods &method) const {
495 2542 : exporter.saveResult(*layer_node_props, method, this);
496 2542 : layer->exportTo(exporter, method);
497 2542 : }
498 :
499 1 : void LayerNode::read(std::ifstream &file, bool opt_var,
500 : ml::train::ExecutionMode mode, bool fsu,
501 : size_t start_offset, bool read_from_offset, int file_fd) {
502 2 : NNTR_THROW_IF(!run_context, std::runtime_error)
503 : << __func__ << " layer needs to be finalized first!";
504 0 : getLayer()->read(file, *run_context, opt_var, mode,
505 0 : (getTrainable() && mode == ml::train::ExecutionMode::TRAIN),
506 : getWeightDataType(), fsu, start_offset, read_from_offset,
507 : file_fd);
508 0 : }
509 :
510 0 : void LayerNode::read(ReadSource src, bool opt_var,
511 : ml::train::ExecutionMode mode, bool fsu,
512 : size_t start_offset, bool read_from_offset) {
513 0 : NNTR_THROW_IF(!run_context, std::runtime_error)
514 : << __func__ << " layer needs to be finalized first!";
515 0 : getLayer()->read(src, *run_context, opt_var, mode,
516 0 : (getTrainable() && mode == ml::train::ExecutionMode::TRAIN),
517 : getWeightDataType(), fsu, start_offset, read_from_offset);
518 0 : }
519 :
520 3135 : void LayerNode::save(std::ofstream &file, bool opt_var,
521 : ml::train::ExecutionMode mode) const {
522 3136 : NNTR_THROW_IF(!run_context, std::runtime_error)
523 : << __func__ << " layer needs to be finalized first!";
524 6268 : getLayer()->save(file, *run_context, opt_var, mode,
525 3134 : (getTrainable() && mode == ml::train::ExecutionMode::TRAIN),
526 : getWeightDataType());
527 3134 : }
528 :
529 93 : void LayerNode::clearOptVar() {
530 94 : NNTR_THROW_IF(!run_context, std::runtime_error)
531 : << __func__ << " layer needs to be finalized first!";
532 148 : for (unsigned int i = 0; i < run_context->getNumWeights(); ++i) {
533 56 : if (run_context->isGradientLastAccess(i) && getTrainable()) {
534 : /// @note read optimizer variables
535 150 : for (unsigned int j = 0; j < run_context->getNumWeightOptVar(i); ++j) {
536 96 : run_context->getWeightOptVar(i, j).initialize();
537 : }
538 : }
539 : }
540 92 : }
541 :
542 : /**
543 : * @brief Finalize creating the layer node
544 : */
545 4575 : InitLayerContext LayerNode::finalize(const std::vector<TensorDim> &input_dims,
546 : std::array<std::string, 3> tensor_type,
547 : ml::train::ExecutionMode mode) {
548 : // auto get_tensor_datatype = [](const std::string ty) -> TensorDim::DataType
549 : // { return from_string(ty);
550 : // };
551 :
552 4575 : if (run_context)
553 : throw std::runtime_error(
554 1 : "Trying to finalizing a layer which is already finalized in layer: " +
555 3 : getName());
556 :
557 : std::vector<TensorDim> actual_input_dims;
558 : auto &prop_dims = std::get<std::vector<props::InputShape>>(*layer_node_props);
559 : auto &prop_in_layers =
560 : std::get<std::vector<props::InputConnection>>(*layer_node_props);
561 :
562 : /** prepare input dimensions */
563 4574 : if (!input_dims.empty()) {
564 3435 : actual_input_dims = input_dims;
565 3435 : if (hasInputShapeProperty()) {
566 : std::vector<TensorDim> actual_prop_dims(prop_dims.begin(),
567 0 : prop_dims.end());
568 : /// if prop_dims exist, check if it's same with given input_dims
569 0 : NNTR_THROW_IF(input_dims != actual_prop_dims, std::invalid_argument)
570 : << "calculated input dimension is different from given input_shape "
571 : "property";
572 0 : for (auto &d : actual_prop_dims) {
573 0 : d.setDataType(
574 : str_converter<enum_class_prop_tag, nntrainer::TensorDataTypeInfo>::
575 : from_string(tensor_type[2]));
576 0 : d.setFormat(
577 : str_converter<enum_class_prop_tag, nntrainer::TensorFormatInfo>::
578 : from_string(tensor_type[0]));
579 : }
580 0 : }
581 : } else {
582 1141 : NNTR_THROW_IF(!hasInputShapeProperty(), std::invalid_argument)
583 : << "if input dims not given, input shapes must be given by the user as "
584 : "property";
585 : /// arguably, below check can go away
586 1137 : NNTR_THROW_IF((prop_dims.size() != prop_in_layers.size()) &&
587 : (prop_dims.size() != 1 || !prop_in_layers.empty()),
588 : std::invalid_argument)
589 : << "input shapes must be one if connection is not given but given "
590 : "dimesions size of: "
591 : << prop_dims.size();
592 : actual_input_dims =
593 1137 : std::vector<TensorDim>(prop_dims.begin(), prop_dims.end());
594 2330 : for (auto &d : actual_input_dims) {
595 : /// Input Tensor type of input layer needs to be float.
596 2386 : d.setDataType(
597 : str_converter<enum_class_prop_tag,
598 : nntrainer::TensorDataTypeInfo>::from_string("FP32"));
599 1193 : d.setFormat(
600 : str_converter<enum_class_prop_tag, nntrainer::TensorFormatInfo>::
601 : from_string(tensor_type[0]));
602 : }
603 : }
604 :
605 4572 : NNTR_THROW_IF(actual_input_dims.size() < getNumInputConnections(),
606 : std::invalid_argument)
607 : << "number of input dimensions must be equal or larger "
608 0 : << "than number of input connections, node name: " << getName()
609 : << " num input dims: " << input_dims.size()
610 0 : << " num connections: " << getNumInputConnections();
611 :
612 : /** manipulate layers if required */
613 9144 : if (getType() != TimeDistLayer::type && getDistribute()) {
614 2 : std::unique_ptr<TimeDistLayer> dlayer(new TimeDistLayer());
615 : NNTR_THROW_IF(!dlayer, std::invalid_argument)
616 : << "Error creating time distribution layer";
617 : dlayer->setDistLayer(std::move(layer));
618 : layer = std::move(dlayer);
619 : }
620 :
621 9144 : const auto &scope = getSharedFrom().empty() ? getName() : getSharedFrom();
622 : float max_norm = 0.0;
623 : float loss_scale = 1.0;
624 4572 : if (!std::get<props::ClipGradByGlobalNorm>(*layer_node_props).empty())
625 16 : max_norm = std::get<props::ClipGradByGlobalNorm>(*layer_node_props).get();
626 :
627 4572 : if (!std::get<props::LossScaleForMixed>(*layer_node_props).empty())
628 4113 : loss_scale = std::get<props::LossScaleForMixed>(*layer_node_props).get();
629 :
630 4572 : if (!std::get<props::ComputeEngine>(*layer_node_props).empty()) {
631 0 : compute_engine = std::get<props::ComputeEngine>(*layer_node_props).get();
632 : }
633 :
634 4572 : if (!std::get<props::Packed>(*layer_node_props).empty()) {
635 4572 : bool isPacked = std::get<props::Packed>(*layer_node_props);
636 4572 : if (!isPacked) {
637 : // set weight type = activation type
638 : tensor_type[1] = tensor_type[2];
639 : }
640 : }
641 :
642 4572 : if (!std::get<props::WeightDtype>(*layer_node_props).empty()) {
643 : // set weight type = tensor_data_type
644 8 : tensor_type[1] = to_string(std::get<props::WeightDtype>(*layer_node_props));
645 : }
646 :
647 : std::vector<bool> out_info;
648 4572 : out_info.reserve(output_connections.size());
649 : std::transform(output_connections.begin(), output_connections.end(),
650 : std::back_inserter(out_info), [](auto &con) { return !!con; });
651 :
652 4572 : if (requireLabel() && out_info.empty()) {
653 : /// as we are using output Grad to save label, add fake out info if it's
654 : /// label. This should be substituted to the proper label management
655 630 : out_info.push_back(true);
656 : }
657 :
658 : auto context = InitLayerContext(
659 : actual_input_dims, out_info, getInPlaceType() != InPlaceType::NONE,
660 13724 : getName(), scope, max_norm, tensor_type, loss_scale, mode, compute_engine);
661 :
662 4571 : layer->finalize(context);
663 :
664 : #ifdef ENABLE_TEST
665 9132 : init_context = std::make_unique<InitLayerContext>(context);
666 : #endif // ENABLE_TEST
667 :
668 : #ifdef PROFILE
669 : auto profile_name = [this](const char *suffix) {
670 : return getName() + suffix + "(" + getType() + ")";
671 : };
672 : #endif
673 :
674 : PROFILE_TIME_REGISTER_EVENT(forward_event_key, profile_name(FORWARD_SUFFIX));
675 : PROFILE_TIME_REGISTER_EVENT(calc_deriv_event_key,
676 : profile_name(CALC_DERIV_SUFFIX));
677 : PROFILE_TIME_REGISTER_EVENT(calc_grad_event_key,
678 : profile_name(CALC_GRAD_SUFFIX));
679 :
680 4566 : return context;
681 4579 : }
682 :
683 : /**
684 : * @brief Refinalize creating the layer node
685 : */
686 : InitLayerContext
687 28 : LayerNode::refinalize(const std::vector<TensorDim> &input_dims) {
688 : std::vector<TensorDim> actual_input_dims;
689 : auto &prop_dims = std::get<std::vector<props::InputShape>>(*layer_node_props);
690 : auto &prop_in_layers =
691 : std::get<std::vector<props::InputConnection>>(*layer_node_props);
692 :
693 : /** prepare input dimensions */
694 28 : if (!input_dims.empty()) {
695 27 : actual_input_dims = input_dims;
696 27 : if (hasInputShapeProperty()) {
697 : std::vector<TensorDim> actual_prop_dims(prop_dims.begin(),
698 0 : prop_dims.end());
699 : /// if prop_dims exist, check if it's same with given input_dims
700 0 : NNTR_THROW_IF(input_dims != actual_prop_dims, std::invalid_argument)
701 : << "calculated input dimension is different from given input_shape "
702 : "property";
703 0 : }
704 : } else {
705 1 : NNTR_THROW_IF(!hasInputShapeProperty(), std::invalid_argument)
706 : << "if input dims not given, input shapes must be given by the user as "
707 : "property";
708 : /// arguably, below check can go away
709 1 : NNTR_THROW_IF((prop_dims.size() != prop_in_layers.size()) &&
710 : (prop_dims.size() != 1 || !prop_in_layers.empty()),
711 : std::invalid_argument)
712 : << "input shapes must be one if connection is not given but given "
713 : "dimesions size of: "
714 : << prop_dims.size();
715 : actual_input_dims =
716 1 : std::vector<TensorDim>(prop_dims.begin(), prop_dims.end());
717 : }
718 :
719 28 : NNTR_THROW_IF(actual_input_dims.size() < getNumInputConnections(),
720 : std::invalid_argument)
721 : << "number of input dimensions must be equal or larger "
722 0 : << "than number of input connections, node name: " << getName()
723 : << " num input dims: " << input_dims.size()
724 0 : << " num connections: " << getNumInputConnections();
725 :
726 : /** manipulate layers if required */
727 56 : if (getType() != TimeDistLayer::type && getDistribute()) {
728 0 : std::unique_ptr<TimeDistLayer> dlayer(new TimeDistLayer());
729 : NNTR_THROW_IF(!dlayer, std::invalid_argument)
730 : << "Error creating time distribution layer";
731 : dlayer->setDistLayer(std::move(layer));
732 : layer = std::move(dlayer);
733 : }
734 :
735 56 : const auto &scope = getSharedFrom().empty() ? getName() : getSharedFrom();
736 : float max_norm = 0.0;
737 28 : if (!std::get<props::ClipGradByGlobalNorm>(*layer_node_props).empty())
738 0 : max_norm = std::get<props::ClipGradByGlobalNorm>(*layer_node_props).get();
739 :
740 : std::vector<bool> out_info;
741 28 : out_info.reserve(output_connections.size());
742 : std::transform(output_connections.begin(), output_connections.end(),
743 : std::back_inserter(out_info), [](auto &con) { return !!con; });
744 :
745 28 : if (requireLabel() && out_info.empty()) {
746 : /// as we are using output Grad to save label, add fake out info if it's
747 : /// label. This should be substituted to the proper label management
748 1 : out_info.push_back(true);
749 : }
750 :
751 : auto context = InitLayerContext(actual_input_dims, out_info,
752 : getInPlaceType() != InPlaceType::NONE,
753 56 : getName(), scope, max_norm);
754 :
755 28 : layer->finalize(context);
756 :
757 : #ifdef ENABLE_TEST
758 56 : init_context = std::make_unique<InitLayerContext>(context);
759 : #endif // ENABLE_TEST
760 :
761 : #ifdef PROFILE
762 : auto profile_name = [this](const char *suffix) {
763 : return getName() + suffix + "(" + getType() + ")";
764 : };
765 : #endif
766 :
767 : PROFILE_TIME_REGISTER_EVENT(forward_event_key, profile_name(FORWARD_SUFFIX));
768 : PROFILE_TIME_REGISTER_EVENT(calc_deriv_event_key,
769 : profile_name(CALC_DERIV_SUFFIX));
770 : PROFILE_TIME_REGISTER_EVENT(calc_grad_event_key,
771 : profile_name(CALC_GRAD_SUFFIX));
772 :
773 28 : return context;
774 84 : }
775 :
776 : /**
777 : * @brief Forward Propagation of a layer
778 : */
779 27303 : void LayerNode::forwarding(bool training) {
780 27303 : loss->set(run_context->getRegularizationLoss());
781 :
782 : PROFILE_TIME_START(forward_event_key);
783 27303 : if (reStoreData()) {
784 0 : if (getInPlaceType() == InPlaceType::NONE) {
785 0 : for (unsigned int i = 0; i < run_context->getNumOutputs(); ++i) {
786 0 : run_context->getOutput(i).setValue(0);
787 0 : if (!run_context->getOutputGradUnsafe(i).isValid())
788 0 : run_context->getOutputGradUnsafe(i).setValue(0);
789 : }
790 0 : for (unsigned int i = 0; i < run_context->getNumWeights(); ++i) {
791 0 : if (run_context->weightHasGradient(i)) {
792 0 : run_context->getWeightGrad(i).setValue(0);
793 : }
794 : }
795 : }
796 : }
797 :
798 27303 : layer->forwarding(*run_context, training);
799 : reStoreData(false);
800 : PROFILE_TIME_END(forward_event_key);
801 54606 : TRACE_MEMORY() << getName() + ": F";
802 54606 : TRACE_TIME() << getName() + ": F";
803 :
804 : #ifdef DEBUG
805 : if (!run_context->validate(getNumInputConnections() == 0, !requireLabel()))
806 : throw std::runtime_error("Running forwarding() layer " + getName() +
807 : " invalidated the context.");
808 : #endif
809 :
810 : /** add loss only for loss layers */
811 27303 : if (requireLabel())
812 6844 : loss->set(*loss + run_context->getLoss());
813 27303 : }
814 :
815 : /**
816 : * @brief Incremental forward Propagation of a layer
817 : */
818 0 : void LayerNode::incremental_forwarding(unsigned int from, unsigned int to,
819 : bool training) {
820 0 : loss->set(run_context->getRegularizationLoss());
821 : PROFILE_TIME_START(forward_event_key);
822 : // std::cerr << getType() << "\n";
823 0 : layer->incremental_forwarding(*run_context, from, to, training);
824 : PROFILE_TIME_END(forward_event_key);
825 0 : TRACE_MEMORY() << getName() + ": F";
826 0 : TRACE_TIME() << getName() + ": F";
827 :
828 : #ifdef DEBUG
829 : if (!run_context->validate(getNumInputConnections() == 0, !requireLabel()))
830 : throw std::runtime_error("Running forwarding() layer " + getName() +
831 : " invalidated the context.");
832 : #endif
833 :
834 : /** add loss only for loss layers */
835 0 : if (requireLabel())
836 0 : loss->set(*loss + run_context->getLoss());
837 0 : }
838 :
839 : /**
840 : * @brief calc the derivative to be passed to the previous layer
841 : */
842 9774 : void LayerNode::calcDerivative() {
843 : PROFILE_TIME_START(calc_deriv_event_key);
844 : PROFILE_MEM_ANNOTATE("CalcDerivative: " + getName());
845 9774 : layer->calcDerivative(*run_context);
846 : PROFILE_TIME_END(calc_deriv_event_key);
847 19548 : TRACE_MEMORY() << getName() + ": CD";
848 19548 : TRACE_TIME() << getName() + ": CD";
849 :
850 : #ifdef DEBUG
851 : if (!run_context->validate(getNumInputConnections() == 0, !requireLabel()))
852 : throw std::runtime_error("Running calcDerivative() layer " + getName() +
853 : " invalidated the context.");
854 : #endif
855 9774 : }
856 :
857 : /**
858 : * @brief Calculate the derivative of a layer
859 : */
860 7467 : void LayerNode::calcGradient() {
861 : PROFILE_TIME_START(calc_grad_event_key);
862 7467 : if (needs_calc_gradient) {
863 : PROFILE_MEM_ANNOTATE("CalcGradient: " + getName());
864 7458 : layer->calcGradient(*run_context);
865 14916 : TRACE_MEMORY() << getName() + ": CG";
866 14916 : TRACE_TIME() << getName() + ": CG";
867 : }
868 : PROFILE_TIME_END(calc_grad_event_key);
869 :
870 : #ifdef DEBUG
871 : if (!run_context->validate(getNumInputConnections() == 0, !requireLabel()))
872 : throw std::runtime_error("Running calcGradient() layer " + getName() +
873 : " invalidated the context.");
874 : #endif
875 7467 : }
876 :
877 : /**
878 : * @brief Set the batch for the layer
879 : */
880 3063 : void LayerNode::setBatch(unsigned int batch) {
881 3064 : NNTR_THROW_IF(!run_context, std::invalid_argument)
882 : << " setting batch not supported before initialization";
883 :
884 3062 : getLayer()->setBatch(*run_context, batch);
885 3062 : }
886 :
887 0 : void LayerNode::updateTensorsByInputDimensions(
888 : std::vector<TensorDim> input_dimensions) {
889 0 : NNTR_THROW_IF(!run_context, std::invalid_argument)
890 : << " update tensors not supported before initialization";
891 :
892 0 : getLayer()->updateTensorsByInputDimensions(*run_context, input_dimensions);
893 0 : }
894 :
895 : /**
896 : * @brief If the current layer can support in-place
897 : */
898 2468 : bool LayerNode::supportInPlace() const {
899 : ///@note below is a quick fix, we need to have a guard that this shouldn't
900 : /// be
901 : /// query until realizeProps has been finalized ( which means we will need
902 : /// another end point to fixate this property )
903 2468 : if (getDistribute()) {
904 : return false;
905 : }
906 2468 : return layer->supportInPlace();
907 : }
908 :
909 : /**
910 : * @brief Get the inplace direction for the layer
911 : */
912 2390 : InPlaceDirection LayerNode::getInPlaceDirection() const {
913 2390 : return layer->getInPlaceDirection();
914 : };
915 :
916 : /**
917 : * @brief Initialize the in-place settings of the layer
918 : * @return InPlaceType
919 : */
920 2565 : InPlaceType LayerNode::initializeInPlace() {
921 2565 : inplace_type = layer->initializeInPlace();
922 2565 : return inplace_type;
923 : }
924 :
925 : /**
926 : * @brief check if this layer requires label to be passed
927 : */
928 49618 : bool LayerNode::requireLabel() const { return getLayer()->requireLabel(); }
929 :
930 : /**
931 : * @brief get loss for the layer
932 : * @return loss of the layer
933 : */
934 37151 : float LayerNode::getLoss() const { return *loss; }
935 :
936 4437 : void LayerNode::configureRunContext(const std::vector<Weight *> &weights,
937 : const std::vector<Var_Grad *> &inputs,
938 : const std::vector<Var_Grad *> &outputs,
939 : const std::vector<Var_Grad *> &tensors,
940 : float loss_scale,
941 : std::shared_ptr<ContextData> ct_data) {
942 8874 : run_context = std::make_unique<RunLayerContext>(
943 4437 : getName(), getTrainable(), 0.0f, getInPlaceType() != InPlaceType::NONE,
944 4437 : loss_scale, ct_data, false, weights, inputs, outputs, tensors);
945 4437 : }
946 :
947 : /**
948 : * @brief Print Options when printing layer info
949 : */
950 : typedef enum {
951 : // clang-format off
952 : PRINT_INST_INFO = (1 << 0), /**< Option to print type & instance address info */
953 : PRINT_SHAPE_INFO = (1 << 1), /**< Option to print shape information, invalid before initiation*/
954 : PRINT_PROP = (1 << 2), /**< Option to print properties */
955 : PRINT_PROP_META = (1 << 3), /**< Option to print properties that describe meta info
956 : e.g) layer activation type for non-activation layer. */
957 : PRINT_WEIGHTS = (1 << 4), /**< Option to print weights */
958 : PRINT_METRIC = (1 << 5) /**< Option to print metrics (currently loss only) */
959 : // clang-format on
960 : } PrintOption;
961 :
962 28 : void LayerNode::printPreset(std::ostream &out, PrintPreset preset) {
963 : unsigned int flags = 0;
964 :
965 28 : switch (preset) {
966 4 : case PrintPreset::PRINT_ALL:
967 : flags = PRINT_WEIGHTS | PRINT_METRIC;
968 : /// fall through intended
969 4 : case PrintPreset::PRINT_SUMMARY_META:
970 4 : flags |= PRINT_PROP_META;
971 : /// fall through intended
972 4 : case PrintPreset::PRINT_SUMMARY:
973 4 : flags |= PRINT_INST_INFO | PRINT_SHAPE_INFO | PRINT_PROP | PRINT_PROP_META;
974 : break;
975 : case PrintPreset::PRINT_NONE:
976 : return;
977 0 : default:
978 0 : throw ::std::invalid_argument("undefined preset given");
979 : }
980 4 : print(out, flags);
981 : }
982 :
983 476 : void LayerNode::remapIdentifiers(std::function<void(std::string &)> remap_fn) {
984 1 : NNTR_THROW_IF(isFinalized(), std::invalid_argument)
985 : << "cannot remap identifiers after finalized";
986 : auto &name = std::get<props::Name>(*layer_node_props);
987 475 : if (!name.empty()) {
988 475 : remap_fn(name.get());
989 : }
990 :
991 : auto &shared_from = std::get<props::SharedFrom>(*layer_node_props);
992 475 : if (!shared_from.empty()) {
993 274 : remap_fn(shared_from.get());
994 : }
995 :
996 : /** remap connections without touching index */
997 475 : remapConnections(
998 1081 : [&remap_fn](std::string &name, unsigned &_) { remap_fn(name); });
999 475 : }
1000 :
1001 19241 : void LayerNode::remapConnections(
1002 : std::function<void(std::string &, unsigned &)> remap_fn) {
1003 2 : NNTR_THROW_IF(isFinalized(), std::invalid_argument)
1004 : << "cannot remap identifiers after finalized";
1005 :
1006 : auto &input_conns =
1007 : std::get<std::vector<props::InputConnection>>(*layer_node_props);
1008 :
1009 40271 : for (auto &input_layer : input_conns) {
1010 21032 : auto &name = input_layer.get().getName();
1011 21032 : auto &idx = input_layer.get().getIndex();
1012 : remap_fn(name, idx);
1013 : }
1014 :
1015 19239 : for (auto &output_layer : output_connections) {
1016 0 : if (output_layer == nullptr) {
1017 0 : continue;
1018 : }
1019 :
1020 : auto &name = output_layer->getName();
1021 : auto &idx = output_layer->getIndex();
1022 : remap_fn(name, idx);
1023 : }
1024 19239 : }
1025 :
1026 385 : std::unique_ptr<LayerNode> LayerNode::cloneConfiguration() {
1027 1 : NNTR_THROW_IF(isFinalized(), std::invalid_argument)
1028 : << "It is prohibited to clone configuration";
1029 384 : Exporter e;
1030 384 : exportTo(e, ml::train::ExportMethods::METHOD_STRINGVECTOR);
1031 384 : e.saveResult(*layer_node_props_realization,
1032 : ml::train::ExportMethods::METHOD_STRINGVECTOR, this);
1033 384 : auto props = e.getResult<ml::train::ExportMethods::METHOD_STRINGVECTOR>();
1034 :
1035 : std::vector<std::string> key_val_props;
1036 384 : key_val_props.reserve(props->size());
1037 5004 : for (auto &entry : *props) {
1038 9240 : key_val_props.push_back(entry.first + "=" + entry.second);
1039 : }
1040 :
1041 768 : return createLayerNode(getType(), key_val_props);
1042 384 : }
1043 :
1044 4 : void LayerNode::printShapeInfo(std::ostream &out) {
1045 8 : for (unsigned int idx = 0; idx < getNumInputs(); ++idx) {
1046 4 : out << "input " << run_context->getInput(idx).getDim();
1047 : }
1048 6 : for (unsigned int idx = 0; idx < getNumWeights(); idx++) {
1049 2 : out << "weight " << run_context->getWeight(idx).getDim();
1050 : }
1051 8 : for (unsigned int idx = 0; idx < getNumOutputs(); ++idx) {
1052 4 : out << "output " << run_context->getOutput(idx).getDim();
1053 : }
1054 4 : }
1055 :
1056 4 : void LayerNode::printMetric(std::ostream &out) {
1057 8 : out << "Layer loss value: " << getLoss() << "\n";
1058 4 : }
1059 :
1060 4 : void LayerNode::print(std::ostream &out, unsigned int flags) {
1061 : /** @todo properly move print to LayerNode */
1062 4 : if (flags & PRINT_INST_INFO) {
1063 4 : out << "===================";
1064 8 : if (getName().empty())
1065 0 : printInstance(out, this);
1066 : else
1067 8 : out << "<" << getName() << ">" << std::endl;
1068 :
1069 8 : out << "Layer Type: " << getType() << std::endl;
1070 : }
1071 :
1072 4 : if (flags & PRINT_SHAPE_INFO) {
1073 4 : if (run_context) {
1074 : out << "======shape information: " << std::endl;
1075 4 : printShapeInfo(out);
1076 : }
1077 : }
1078 :
1079 4 : if (flags & PRINT_PROP_META) {
1080 : out << "======meta properties: " << std::endl;
1081 : /** @todo print local and layer properties with node_exporter */
1082 4 : nntrainer::Exporter e;
1083 4 : getLayer()->exportTo(e, ml::train::ExportMethods::METHOD_STRINGVECTOR);
1084 : auto prop_meta =
1085 4 : e.getResult<ml::train::ExportMethods::METHOD_STRINGVECTOR>();
1086 4 : if (prop_meta != nullptr) {
1087 15 : for (unsigned int i = 0; i < prop_meta->size(); ++i) {
1088 12 : out << (*prop_meta)[i].first << ": " << (*prop_meta)[i].second << "\n";
1089 : }
1090 : }
1091 4 : }
1092 :
1093 4 : if (flags & PRINT_PROP) {
1094 : out << "======properties: " << std::endl;
1095 : /** @todo print local and layer properties with node_exporter */
1096 : }
1097 :
1098 4 : if (flags & PRINT_WEIGHTS) {
1099 4 : if (run_context) {
1100 : out << "======weights: " << std::endl;
1101 6 : for (unsigned int idx = 0; idx < getNumWeights(); idx++) {
1102 2 : out << run_context->getWeight(idx);
1103 : }
1104 : }
1105 : }
1106 :
1107 4 : if (flags & PRINT_METRIC) {
1108 : out << "======metrics: " << std::endl;
1109 4 : printMetric(out);
1110 : }
1111 4 : };
1112 : }; // namespace nntrainer
|
