Line data Source code
1 : // SPDX-License-Identifier: Apache-2.0
2 : /**
3 : * Copyright (C) 2021 Jihoon Lee <jhoon.it.lee@samsung.com>
4 : *
5 : * @file common_properties.h
6 : * @date 09 April 2021
7 : * @brief This file contains list of common properties widely used across
8 : * layers
9 : * @see https://github.com/nnstreamer/nntrainer
10 : * @author Jihoon Lee <jhoon.it.lee@samsung.com>
11 : * @bug No known bugs except for NYI items
12 : */
13 : #ifndef __COMMON_PROPERTIES_H__
14 : #define __COMMON_PROPERTIES_H__
15 :
16 : #include <array>
17 : #include <fstream>
18 : #include <string>
19 :
20 : #include <base_properties.h>
21 : #include <common.h>
22 : #include <connection.h>
23 : #include <tensor.h>
24 : #include <tensor_wrap_specs.h>
25 :
26 : namespace nntrainer {
27 :
28 : /**
29 : * @brief Enumeration of activation function type
30 : * @note Upon changing this enum, ActivationTypeInfo must be changed
31 : * accordingly
32 : */
33 : enum class ActivationType {
34 : ACT_TANH, /**< tanh */
35 : ACT_SIGMOID, /**< sigmoid */
36 : ACT_RELU, /**< ReLU */
37 : ACT_SWISH, /**< Swish */
38 : ACT_GELU, /**< GELU */
39 : ACT_TANH_GELU, /**< tanh GELU */
40 : ACT_SIGMOID_GELU, /**< sigmoid GELU */
41 : ACT_SOFTMAX, /**< softmax */
42 : ACT_SOFTPLUS, /**< softplus */
43 : ACT_LEAKY_RELU, /**< Leaky ReLU */
44 : ACT_ELU, /**< ELU */
45 : ACT_SELU, /**< SELU */
46 : ACT_MISH, /**< Mish */
47 : ACT_NONE, /**< no op */
48 : ACT_UNKNOWN /**< unknown */
49 : };
50 :
51 : namespace props {
52 :
53 : /**
54 : * @brief Name property, name is an identifier of an object
55 : *
56 : */
57 35053 : class Name : public nntrainer::Property<std::string> {
58 : public:
59 : /**
60 : * @brief Construct a new Name object without a default value
61 : *
62 : */
63 : Name();
64 :
65 : /**
66 : * @brief Construct a new Name object with a default value
67 : *
68 : * @param value value to contrusct the property
69 : */
70 : Name(const std::string &value);
71 :
72 : static constexpr const char *key = "name"; /**< unique key to access */
73 : using prop_tag = str_prop_tag; /**< property type */
74 :
75 : /**
76 : * @brief Name setter
77 : *
78 : * @param value value to set
79 : */
80 : void set(const std::string &value) override;
81 :
82 : /**
83 : * @brief name validator
84 : *
85 : * @param v string to validate
86 : * @retval true if it contains alphanumeric and/or '-', '_', '/'
87 : * @retval false if it is empty or contains non-valid character
88 : */
89 : bool isValid(const std::string &v) const override;
90 : };
91 :
92 : /**
93 : * @brief unit property, unit is used to measure how many weights are there
94 : *
95 : */
96 3229 : class Unit : public PositiveIntegerProperty {
97 : public:
98 : static constexpr const char *key = "unit"; /**< unique key to access */
99 : using prop_tag = uint_prop_tag; /**< property type */
100 : };
101 :
102 : /**
103 : * @brief trainable property, use this to set and check how if certain layer is
104 : * trainable
105 : *
106 : */
107 12382 : class Trainable : public nntrainer::Property<bool> {
108 : public:
109 : /**
110 : * @brief Construct a new Trainable object
111 : *
112 : */
113 10732 : Trainable(bool val = true) : nntrainer::Property<bool>(val) {}
114 : static constexpr const char *key = "trainable";
115 : using prop_tag = bool_prop_tag;
116 : };
117 :
118 : /**
119 : * @brief Tensor Dimension property
120 : *
121 : */
122 74 : class TensorDimension : public TensorDimProperty {
123 : public:
124 : static constexpr const char *key = "dim"; /**< unique key to access */
125 : using prop_tag = dimension_prop_tag; /**< property type */
126 : };
127 :
128 : /**
129 : * @brief Inplace operation property
130 : *
131 : */
132 447 : class InPlaceProp : public nntrainer::Property<bool> {
133 : public:
134 : static constexpr const char *key = "inplace"; /**< unique key to access */
135 : using prop_tag = bool_prop_tag; /**< property type */
136 : };
137 :
138 : /**
139 : * @brief Inplace direction property
140 : *
141 : */
142 326 : class InPlaceDirectionProp : public nntrainer::Property<std::string> {
143 : public:
144 : static constexpr const char *key =
145 : "inplace_direction"; /**< unique key to access */
146 : using prop_tag = str_prop_tag; /**< property type */
147 : };
148 :
149 : /**
150 : * @brief trainable property, use this to set and check how if certain layer is
151 : * trainable
152 : *
153 : */
154 12382 : class Packed : public nntrainer::Property<bool> {
155 : public:
156 : /**
157 : * @brief Construct a new Trainable object
158 : * if it is true, then weight type always follows tensor_type[1]( Global
159 : * Weight Type ). if it is false, the weight type follows tensor_type[2]
160 : * (Global Activation Type)
161 : */
162 6191 : Packed(bool val = true) : nntrainer::Property<bool>(val) {}
163 : static constexpr const char *key = "packed";
164 : using prop_tag = bool_prop_tag;
165 : };
166 :
167 : /**
168 : * @brief DisableBias to disable the bias
169 : *
170 : */
171 : class DisableBias : public nntrainer::Property<bool> {
172 : public:
173 : /**
174 : * @brief Construct a DisableBias object
175 : *
176 : */
177 2086 : DisableBias(bool val = false) : nntrainer::Property<bool>(val) {}
178 : using prop_tag = bool_prop_tag;
179 : static constexpr const char *key = "disable_bias";
180 : };
181 :
182 : /**
183 : * @brief Integrate bias_ih and bias_hh to bias_h to use only 1 bias (Used in
184 : * rnn variant)
185 : *
186 : */
187 1130 : class IntegrateBias : public nntrainer::Property<bool> {
188 : public:
189 : /**
190 : * @brief Construct a IntegrateBias object
191 : *
192 : */
193 662 : IntegrateBias(bool val = false) : nntrainer::Property<bool>(val) {}
194 : using prop_tag = bool_prop_tag;
195 : static constexpr const char *key = "integrate_bias";
196 : };
197 :
198 : /**
199 : * @brief Normalization property, normalize the input to be in range [0, 1] if
200 : * true
201 : *
202 : */
203 1432 : class Normalization : public nntrainer::Property<bool> {
204 : public:
205 : /**
206 : * @brief Construct a new Normalization object
207 : *
208 : */
209 : Normalization(bool value = false);
210 : static constexpr const char *key = "normalization";
211 : using prop_tag = bool_prop_tag;
212 : };
213 :
214 : /**
215 : * @brief Standardization property, standardization standardize the input
216 : * to be mean 0 and std 1 if true
217 : *
218 : */
219 1432 : class Standardization : public nntrainer::Property<bool> {
220 : public:
221 : /**
222 : * @brief Construct a new Standardization object
223 : *
224 : */
225 : Standardization(bool value = false);
226 : static constexpr const char *key = "standardization";
227 : using prop_tag = bool_prop_tag;
228 : };
229 :
230 : /**
231 : * @brief Connection prop tag type
232 : *
233 : */
234 : struct connection_prop_tag {};
235 :
236 : /**
237 : * @brief InputSpec property, this defines connection specification of an input
238 : *
239 : */
240 12228 : class InputConnection : public nntrainer::Property<Connection> {
241 : public:
242 : /**
243 : * @brief Construct a new Input Spec object
244 : *
245 : */
246 : InputConnection();
247 :
248 : /**
249 : * @brief Construct a new Input Spec object
250 : *
251 : * @param value default value of a input spec
252 : */
253 : InputConnection(const Connection &value);
254 : static constexpr const char *key =
255 : "input_layers"; /**< unique key to access */
256 : using prop_tag = connection_prop_tag; /**< property type */
257 : };
258 :
259 : /**
260 : * @brief Epsilon property, this is used to avoid divide by zero
261 : *
262 : */
263 194 : class Epsilon : public nntrainer::Property<float> {
264 :
265 : public:
266 : /**
267 : * @brief Construct a new Epsilon object with a default value 0.001
268 : *
269 : */
270 : Epsilon(float value = 0.001);
271 : static constexpr const char *key = "epsilon"; /**< unique key to access */
272 : using prop_tag = float_prop_tag; /**< property type */
273 :
274 : /**
275 : * @brief Epsilon validator
276 : *
277 : * @param value float to validate
278 : * @retval true if it is greater or equal than 0.0
279 : * @retval false if it is smaller than 0.0
280 : */
281 : bool isValid(const float &value) const override;
282 : };
283 :
284 : /**
285 : * @brief Exponent property, this is used for pow operation
286 : *
287 : */
288 40 : class Exponent : public nntrainer::Property<float> {
289 :
290 : public:
291 : /**
292 : * @brief Construct a new Exponent object
293 : *
294 : */
295 : Exponent(float value = 1.0f);
296 : static constexpr const char *key = "exponent"; /**< unique key to access */
297 : using prop_tag = float_prop_tag; /**< property type */
298 : };
299 :
300 : /**
301 : * @brief Momentum property, moving average in batch normalization layer
302 : *
303 : */
304 2 : class Momentum : public nntrainer::Property<float> {
305 :
306 : public:
307 : /**
308 : * @brief Construct a new Momentum object with a default value 0.99
309 : *
310 : */
311 : Momentum(float value = 0.99);
312 : static constexpr const char *key = "momentum"; /**< unique key to access */
313 : using prop_tag = float_prop_tag; /**< property type */
314 :
315 : /**
316 : * @brief Momentum validator
317 : *
318 : * @param value float to validate
319 : * @retval true if it is greater than 0.0 and smaller than 1.0
320 : * @retval false if it is smaller or equal than 0.0
321 : * or greater or equal than 1.0
322 : */
323 : bool isValid(const float &value) const override;
324 : };
325 :
326 : /**
327 : * @brief split number property, split number indicates how many numbers of outs
328 : * are generated by spliting the input dimension
329 : *
330 : */
331 78 : class SplitNumber : public PositiveIntegerProperty {
332 : public:
333 : static constexpr const char *key =
334 : "split_number"; /**< unique key to access */
335 : using prop_tag = uint_prop_tag; /**< property type */
336 : };
337 :
338 : /**
339 : * @brief Axis property, idx in the dimension
340 : *
341 : */
342 708 : class Axis : public nntrainer::PositiveIntegerProperty {
343 : public:
344 : static constexpr const char *key = "axis"; /**< unique key to access */
345 : using prop_tag = uint_prop_tag; /**< property type */
346 :
347 : /**
348 : * @brief check if given value is valid
349 : *
350 : * @param v value to check
351 : * @retval true if it is greater equal to 0 and smaller than
352 : * ml::train::TensorDim::MAXDIM
353 : * @retval false if it is smaller than 0 or greater than
354 : * ml::train::TensorDim::MAXDIM
355 : */
356 : bool isValid(const unsigned int &value) const override;
357 : };
358 :
359 : /**
360 : * @brief StartDimension property, start dimension to be flatten
361 : *
362 : */
363 320 : class StartDimension : public Axis {
364 : public:
365 : StartDimension(unsigned int value = 1);
366 : static constexpr const char *key = "start_dimension";
367 : using prop_tag = uint_prop_tag;
368 :
369 : /**
370 : * @brief check if given value is valid
371 : *
372 : * @param v value to check
373 : * @retval true if it is greater than 0 and smaller than
374 : * ml::train::TensorDim::MAXDIM
375 : * @retval false if it is smaller or equal to 0 or greater than
376 : * ml::train::TensorDim::MAXDIM
377 : */
378 : bool isValid(const unsigned int &value) const override;
379 : };
380 :
381 : /**
382 : * @brief EndDimension property, end dimension to be flatten
383 : *
384 : */
385 320 : class EndDimension : public Axis {
386 : public:
387 : EndDimension(unsigned int value = ml::train::TensorDim::MAXDIM - 1);
388 : static constexpr const char *key = "end_dimension";
389 : using prop_tag = uint_prop_tag;
390 :
391 : /**
392 : * @brief check if given value is valid
393 : *
394 : * @param v value to check
395 : * @retval true if it is greater than 0 and smaller than
396 : * ml::train::TensorDim::MAXDIM
397 : * @retval false if it is smaller or equal to 0 or greater than
398 : * ml::train::TensorDim::MAXDIM
399 : */
400 : bool isValid(const unsigned int &value) const override;
401 : };
402 :
403 : /**
404 : * @brief StartIndex property, start index to be slicing
405 : *
406 : */
407 0 : class StartIndex : public PositiveIntegerProperty {
408 : public:
409 : static constexpr const char *key = "start_index"; /**< unique key to access */
410 : using prop_tag = uint_prop_tag; /**< property type */
411 : };
412 :
413 : /**
414 : * @brief EndIndex property, end index to be slicing
415 : *
416 : */
417 0 : class EndIndex : public PositiveIntegerProperty {
418 : public:
419 : static constexpr const char *key = "end_index"; /**< unique key to access */
420 : using prop_tag = uint_prop_tag; /**< property type */
421 : };
422 :
423 : /**
424 : * @brief SplitDimension property, dimension along which to split the input
425 : *
426 : */
427 150 : class SplitDimension : public Axis {
428 : public:
429 : /**
430 : * @brief check if given value is valid
431 : *
432 : * @param v value to check
433 : * @retval true if it is greater than 0 and smaller than
434 : * ml::train::TensorDim::MAXDIM
435 : * @retval false if it is smaller or equal to 0 or greater than
436 : * ml::train::TensorDim::MAXDIM
437 : */
438 : bool isValid(const unsigned int &value) const override;
439 : };
440 :
441 : /**
442 : * @brief ConcatDimension property, dimension along which to concat the input
443 : *
444 : */
445 132 : class ConcatDimension : public SplitDimension {};
446 :
447 : /**
448 : * @brief ReduceDimension property, dimension along which to reduce the input
449 : *
450 : */
451 59 : class ReduceDimension : public SplitDimension {};
452 :
453 : /**
454 : * @brief FilterSize property, filter size is used to measure how many filters
455 : * are there
456 : *
457 : */
458 464 : class FilterSize : public nntrainer::PositiveIntegerProperty {
459 : public:
460 : static constexpr const char *key = "filters"; /**< unique key to access */
461 : using prop_tag = uint_prop_tag; /**< property type */
462 : };
463 :
464 : /**
465 : * @brief KernelSize property, kernel size is used to measure the filter size
466 : *
467 : */
468 1536 : class KernelSize : public nntrainer::PositiveIntegerProperty {
469 : public:
470 : static constexpr const char *key = "kernel_size"; /**< unique key to access */
471 : using prop_tag = uint_prop_tag; /**< property type */
472 : };
473 :
474 : /**
475 : * @brief PoolSize property, pool size is used to measure the pooling size
476 : *
477 : */
478 299 : class PoolSize : public nntrainer::PositiveIntegerProperty {
479 : public:
480 : /**
481 : * @brief Construct a new PoolSize object
482 : *
483 : */
484 104 : PoolSize() {}
485 :
486 : /**
487 : * @brief Construct a new PoolSize object with default value
488 : *
489 : */
490 : PoolSize(unsigned int value);
491 : static constexpr const char *key = "pool_size"; /**< unique key to access */
492 : using prop_tag = uint_prop_tag; /**< property type */
493 : };
494 :
495 : /**
496 : * @brief Stride property, stride is used to measure how much it will be slide
497 : * the filter
498 : *
499 : */
500 752 : class Stride : public nntrainer::PositiveIntegerProperty {
501 : public:
502 : /**
503 : * @brief Construct a new Stride object with a default value 1
504 : *
505 : */
506 : Stride(unsigned int value = 1);
507 : static constexpr const char *key = "stride"; /**< unique key to access */
508 : using prop_tag = uint_prop_tag; /**< property type */
509 : };
510 :
511 : /**
512 : * @brief Dilation property, dilation indicates how many space will be inserted
513 : * between kernel element
514 : *
515 : */
516 328 : class Dilation : public nntrainer::PositiveIntegerProperty {
517 : public:
518 : /**
519 : * @brief Construct a new Dilation object with a default value 1
520 : *
521 : */
522 : Dilation(unsigned int value = 1);
523 : static constexpr const char *key = "dilation"; /**< unique key to access */
524 : using prop_tag = uint_prop_tag; /**< property type */
525 : };
526 :
527 : /**
528 : * @brief Padding2D property, this is used to calculate padding2D
529 : * @details Padding2D is saved as a string. Upon calling Padding2D::compute,
530 : * returns std::vector<unsigned int> which has computed padding2Ds, below
531 : * formats are accepted valid
532 : * 1. "same" (case insensitive literal string)
533 : * 2. "valid" (case insensitive literal string)
534 : * 3. "padding2D_all", eg) padding=1
535 : * 4. "padding2D_height, padding2D_width" eg) padding=1,1
536 : * 5. "padding2D_top, padding2D_bottom, padding2D_left, padding2D_right" eg)
537 : * padding=1,1,1,1
538 : *
539 : */
540 640 : class Padding2D final : public nntrainer::Property<std::string> {
541 : public:
542 : /**
543 : * @brief Construct a new Padding2D object
544 : *
545 : */
546 323 : Padding2D(const std::string &value = "valid") :
547 323 : nntrainer::Property<std::string>(value) {} /**< default value if any */
548 : bool isValid(const std::string &v) const override;
549 : static constexpr const char *key = "padding"; /**< unique key to access */
550 : using prop_tag = str_prop_tag; /**< property type */
551 :
552 : /**
553 : * @brief compute actual padding2D from the underlying data
554 : *
555 : * @param input input dimension
556 : * @param kernel kernel dimension
557 : * @param stride stride
558 : * @return std::array<unsigned int, 4> list of unsigned padding
559 : */
560 : std::array<unsigned int, 4>
561 : compute(const TensorDim &input, const TensorDim &kernel,
562 : const std::array<unsigned int, 2> &strides,
563 : const std::array<unsigned int, 2> &dilation);
564 : };
565 :
566 : /**
567 : * @brief Padding1D property, this is used to calculate padding2D
568 : * @details Padding1D is saved as a string. Upon calling Padding1D::compute,
569 : * returns std::vector<unsigned int> which has computed padding1Ds, below
570 : * formats are accepted valid
571 : * 1. "same" (case insensitive literal string)
572 : * 2. "valid" (case insensitive literal string)
573 : * 2. "causal" (case insensitive literal string)
574 : * 3. "padding1d_all", eg) padding=1
575 : * 4. "padding1d_left, padding1d_right" eg) padding=1,1
576 : *
577 : */
578 80 : class Padding1D final : public nntrainer::Property<std::string> {
579 : public:
580 : /**
581 : * @brief Construct a new Padding1D object
582 : *
583 : */
584 40 : Padding1D(const std::string &value = "valid") :
585 40 : nntrainer::Property<std::string>(value) {} /**< default value if any */
586 : bool isValid(const std::string &v) const override;
587 : static constexpr const char *key = "padding"; /**< unique key to access */
588 : using prop_tag = str_prop_tag; /**< property type */
589 :
590 : /**
591 : * @brief compute actual padding1d from the underlying data
592 : *
593 : * @param input input dimension
594 : * @param kernel kernel dimension
595 : * @param stride stride
596 : * @return std::array<unsigned int, 4> list of unsigned padding
597 : */
598 : std::array<unsigned int, 2> compute(const TensorDim &input,
599 : const unsigned int &kernel,
600 : const unsigned int &stride,
601 : const unsigned int &dilation);
602 : };
603 :
604 : /**
605 : * @brief InDim property, in dim is the size of vocabulary in the text data
606 : *
607 : */
608 42 : class InDim : public nntrainer::PositiveIntegerProperty {
609 : public:
610 : static constexpr const char *key = "in_dim"; /**< unique key to access */
611 : using prop_tag = uint_prop_tag; /**< property type */
612 : };
613 :
614 : /**
615 : * @brief OutDim property, out dim is the size of the vector space
616 : * in which words will be embedded
617 : *
618 : */
619 42 : class OutDim : public nntrainer::PositiveIntegerProperty {
620 : public:
621 : static constexpr const char *key = "out_dim"; /**< unique key to access */
622 : using prop_tag = uint_prop_tag; /**< property type */
623 : };
624 :
625 : /**
626 : * @brief Zero idx mask property for embedding where the value of embedding
627 : * will be zero
628 : *
629 : */
630 : class ZeroIdxMask : public nntrainer::Property<unsigned int> {
631 : public:
632 : static constexpr const char *key =
633 : "zero_idx_mask"; /**< unique key to access */
634 : using prop_tag = uint_prop_tag; /**< property type */
635 : };
636 :
637 : /**
638 : * @brief DropOutRate property, this defines drop out specification of layer
639 : *
640 : */
641 599 : class DropOutRate : public nntrainer::Property<float> {
642 :
643 : public:
644 : /**
645 : * @brief Construct a new DropOutRate object with a default value 0.0
646 : *
647 : */
648 565 : DropOutRate(float value = 0.0) : nntrainer::Property<float>(value) {}
649 : static constexpr const char *key =
650 : "dropout_rate"; /**< unique key to access */
651 : using prop_tag = float_prop_tag; /**< property type */
652 :
653 : /**
654 : * @brief DropOutRate validator
655 : *
656 : * @param v float to validate
657 : * @retval true if it is greater or equal than 0.0
658 : * @retval false if it is smaller than 0.0
659 : */
660 : bool isValid(const float &v) const override;
661 : };
662 :
663 : /**
664 : * @brief TranslationFactor property, this defines how far the image is
665 : * translated
666 : *
667 : */
668 0 : class RandomTranslate : public nntrainer::Property<float> {
669 :
670 : public:
671 : static constexpr const char *key =
672 : "random_translate"; /**< unique key to access */
673 : using prop_tag = float_prop_tag; /**< property type */
674 :
675 : /**
676 : * @brief setter
677 : *
678 : * @param value value to set
679 : */
680 : void set(const float &value) override;
681 : };
682 :
683 : /**
684 : * @brief Props containing file path value
685 : *
686 : */
687 515 : class FilePath : public Property<std::string> {
688 : public:
689 : /**
690 : * @brief Construct a new File Path object
691 : */
692 9 : FilePath() : Property<std::string>() {}
693 :
694 : /**
695 : * @brief Construct a new File Path object
696 : *
697 : * @param path path to set
698 : */
699 211 : FilePath(const std::string &path) { set(path); }
700 : static constexpr const char *key = "path"; /**< unique key to access */
701 : using prop_tag = str_prop_tag; /**< property type */
702 :
703 : /**
704 : * @brief check if given value is valid
705 : *
706 : * @param v value to check
707 : * @return bool true if valid
708 : */
709 : bool isValid(const std::string &v) const override;
710 :
711 : /**
712 : * @brief setter
713 : *
714 : * @param v value to set
715 : */
716 : void set(const std::string &v) override;
717 :
718 : /**
719 : * @brief return file size
720 : *
721 : * @return std::ifstream::pos_type size of the file
722 : */
723 : std::ifstream::pos_type file_size();
724 :
725 : private:
726 : std::ifstream::pos_type cached_pos_size;
727 : };
728 :
729 : /**
730 : * @brief Props containing directory path value
731 : *
732 : */
733 0 : class DirPath : public Property<std::string> {
734 : public:
735 : /**
736 : * @brief Construct a new Dir Path object
737 : */
738 0 : DirPath() : Property<std::string>() {}
739 :
740 : /**
741 : * @brief Construct a new Dir Path object
742 : *
743 : * @param path path to set
744 : */
745 0 : DirPath(const std::string &path) { set(path); }
746 : static constexpr const char *key = "dir_path"; /**< unique key to access */
747 : using prop_tag = str_prop_tag; /**< property type */
748 :
749 : /**
750 : * @brief check if given value is valid
751 : *
752 : * @param v value to check
753 : * @return bool true if valid
754 : */
755 : bool isValid(const std::string &v) const override;
756 :
757 : /**
758 : * @brief setter
759 : *
760 : * @param v value to set
761 : */
762 : void set(const std::string &v) override;
763 : };
764 :
765 : /**
766 : * @brief return sequence property, used to check
767 : * whether return only the last output. Return last output if true.
768 : *
769 : */
770 180 : class ReturnSequences : public nntrainer::Property<bool> {
771 : public:
772 : /**
773 : * @brief Construct a new ReturnSequences object
774 : *
775 : */
776 : ReturnSequences(bool value = false);
777 : static constexpr const char *key = "return_sequences";
778 : using prop_tag = bool_prop_tag;
779 : };
780 :
781 : /**
782 : * @brief bidirectional property, used to make bidirectional layers
783 : *
784 : */
785 76 : class Bidirectional : public nntrainer::Property<bool> {
786 : public:
787 : /**
788 : * @brief Construct a new Bidirectional object
789 : *
790 : */
791 : Bidirectional(bool value = false);
792 : static constexpr const char *key = "bidirectional";
793 : using prop_tag = bool_prop_tag;
794 : };
795 :
796 : /**
797 : * @brief Identifiers to locate a connection which should be returned as whole
798 : * used in recurrent realizer
799 : *
800 : */
801 162 : class AsSequence : public Property<Connection> {
802 : public:
803 : static constexpr const char *key = "as_sequence";
804 : using prop_tag = connection_prop_tag;
805 : };
806 :
807 : /**
808 : * @brief Identifiers to locate an **input** connection which should be
809 : * sequenced for the connection
810 : *
811 : */
812 3 : class InputIsSequence : public Name {
813 : public:
814 : static constexpr const char *key = "input_is_sequence";
815 : using prop_tag = str_prop_tag;
816 : };
817 :
818 : /**
819 : * @brief ResetAfter property, apply reset gate after matrix multiplication if
820 : * this property is true. Apply before the multiplication if false. Used in gru,
821 : * grucell.
822 : *
823 : */
824 294 : class ResetAfter : public nntrainer::Property<bool> {
825 :
826 : public:
827 : /**
828 : * @brief Construct a new ResetAfter object with a default value true
829 : *
830 : */
831 147 : ResetAfter(bool value = true) : nntrainer::Property<bool>(value) {}
832 : static constexpr const char *key = "reset_after"; /**< unique key to access */
833 : using prop_tag = bool_prop_tag; /**< property type */
834 : };
835 :
836 : /**
837 : * @brief Number of class
838 : * @todo deprecate this
839 : */
840 14 : class NumClass final : public nntrainer::Property<unsigned int> {
841 : public:
842 : using prop_tag = uint_prop_tag; /**< property type */
843 : static constexpr const char *key = "num_class"; /**< unique key to access */
844 :
845 : /**
846 : * @copydoc nntrainer::Property<unsigned int>::isValid(const unsigned int &v);
847 : */
848 : bool isValid(const unsigned int &v) const override;
849 : };
850 :
851 : /**
852 : * @brief BasicRegularizerConstant property, this defines how much regularize
853 : * the weight
854 : *
855 : */
856 176 : class BasicRegularizerConstant : public nntrainer::Property<float> {
857 :
858 : public:
859 : /**
860 : * @brief Construct a new BasicRegularizerConstant object
861 : *
862 : */
863 : BasicRegularizerConstant(float value = 1.0f);
864 : static constexpr const char *key =
865 : "basic_regularizer_constant"; /**< unique key to access */
866 : using prop_tag = float_prop_tag; /**< property type */
867 :
868 : /**
869 : * @brief check if given value is valid
870 : *
871 : * @param value value to check
872 : * @return bool true if valid
873 : */
874 : bool isValid(const float &value) const override;
875 : };
876 :
877 : /**
878 : * @brief WeightRegularizerConstant property, this defines how much regularize
879 : * the weight
880 : *
881 : */
882 : class WeightRegularizerConstant final : public BasicRegularizerConstant {
883 :
884 : public:
885 : /**
886 : * @brief Construct a new WeightRegularizerConstant object
887 : *
888 : */
889 : WeightRegularizerConstant(float value = 1.0f);
890 : static constexpr const char *key =
891 : "weight_regularizer_constant"; /**< unique key to access */
892 : };
893 :
894 : /**
895 : * @brief WeightDecay property, this defines how much to decay
896 : * the weight
897 : *
898 : */
899 595 : class WeightDecay final : public BasicRegularizerConstant {
900 :
901 : public:
902 : /**
903 : * @brief Construct a new WeightDecay object
904 : *
905 : */
906 : WeightDecay(float value = 0.0f);
907 : static constexpr const char *key =
908 : "weight_decay"; /**< unique key to access */
909 : };
910 :
911 : /**
912 : * @brief BiasDecay property, this defines how much regularize
913 : * the weight
914 : *
915 : */
916 651 : class BiasDecay final : public BasicRegularizerConstant {
917 :
918 : public:
919 : /**
920 : * @brief Construct a new BiasDecay object
921 : *
922 : */
923 : BiasDecay(float value = 0.0f);
924 : static constexpr const char *key = "bias_decay"; /**< unique key to access */
925 : };
926 :
927 : /**
928 : * @brief Output Layer name property which saves a single connection
929 : * (practically, std::vector<InputLayers> is used)
930 : *
931 : */
932 : class OutputLayer : public Name {
933 : public:
934 : /**
935 : * @brief Construct a new Output Layer object
936 : *
937 : */
938 : OutputLayer();
939 :
940 : /**
941 : * @brief Construct a new Output Layer object
942 : *
943 : * @param name name to set
944 : */
945 : OutputLayer(const std::string &name);
946 : static constexpr const char *key = "output_layers";
947 : using prop_tag = str_prop_tag;
948 : };
949 :
950 : /**
951 : * @brief label Layer name property which saves a single
952 : * connection (practically, std::vector<LabelLayers> is used)
953 : *
954 : */
955 168 : class LabelLayer : public Name {
956 : public:
957 : /**
958 : * @brief Construct LabelLayer object
959 : *
960 : */
961 : LabelLayer();
962 :
963 : /**
964 : * @brief Construct LabelLayer with the given name
965 : *
966 : * @param name Name for the input_layers
967 : */
968 : LabelLayer(const std::string &name);
969 : static constexpr const char *key = "label_layers";
970 : using prop_tag = str_prop_tag;
971 : };
972 :
973 : /******** below section is for enumerations ***************/
974 : /**
975 : * @brief Enumeration of activation function type
976 : */
977 : struct ActivationTypeInfo {
978 : using Enum = nntrainer::ActivationType;
979 : static constexpr std::initializer_list<Enum> EnumList = {
980 : Enum::ACT_TANH, Enum::ACT_SIGMOID, Enum::ACT_RELU,
981 : Enum::ACT_SWISH, Enum::ACT_GELU, Enum::ACT_TANH_GELU,
982 : Enum::ACT_SIGMOID_GELU, Enum::ACT_SOFTMAX, Enum::ACT_SOFTPLUS,
983 : Enum::ACT_LEAKY_RELU, Enum::ACT_ELU, Enum::ACT_SELU,
984 : Enum::ACT_MISH, Enum::ACT_NONE, Enum::ACT_UNKNOWN};
985 :
986 : static constexpr const char *EnumStr[] = {
987 : "tanh", "sigmoid", "relu", "swish", "gelu",
988 : "tanh_gelu", "sigmoid_gelu", "softmax", "softplus", "leaky_relu",
989 : "elu", "selu", "mish", "none", "unknown"};
990 : };
991 :
992 : /**
993 : * @brief Activation Enumeration Information
994 : *
995 : */
996 7511 : class Activation final
997 : : public EnumProperty<nntrainer::props::ActivationTypeInfo> {
998 : public:
999 : using prop_tag = enum_class_prop_tag;
1000 : static constexpr const char *key = "activation";
1001 : };
1002 :
1003 : /**
1004 : * @brief HiddenStateActivation Enumeration Information
1005 : *
1006 : */
1007 1324 : class HiddenStateActivation final : public EnumProperty<ActivationTypeInfo> {
1008 : public:
1009 : /**
1010 : * @brief Construct a new HiddenStateActivation object with default value
1011 : * ActivationTypeInfo::Enum::ACT_NONE
1012 : *
1013 : */
1014 : HiddenStateActivation(
1015 : ActivationTypeInfo::Enum value = ActivationTypeInfo::Enum::ACT_NONE);
1016 : using prop_tag = enum_class_prop_tag;
1017 : static constexpr const char *key = "hidden_state_activation";
1018 : };
1019 :
1020 : /**
1021 : * @brief RecurrentActivation Enumeration Information
1022 : *
1023 : */
1024 1132 : class RecurrentActivation final : public EnumProperty<ActivationTypeInfo> {
1025 : public:
1026 : /**
1027 : * @brief Construct a new RecurrentActivation object with default value
1028 : * ActivationTypeInfo::Enum::ACT_NONE
1029 : *
1030 : */
1031 : RecurrentActivation(
1032 : ActivationTypeInfo::Enum value = ActivationTypeInfo::Enum::ACT_NONE);
1033 : using prop_tag = enum_class_prop_tag;
1034 : static constexpr const char *key = "recurrent_activation";
1035 : };
1036 :
1037 : /**
1038 : * @brief Enumeration of tensor initialization type
1039 : */
1040 : struct InitializerInfo {
1041 : using Enum = Initializer;
1042 : static constexpr std::initializer_list<Enum> EnumList = {
1043 : Enum::ZEROS, Enum::ONES, Enum::LECUN_NORMAL,
1044 : Enum::LECUN_UNIFORM, Enum::XAVIER_NORMAL, Enum::XAVIER_UNIFORM,
1045 : Enum::HE_NORMAL, Enum::HE_UNIFORM, Enum::NONE};
1046 :
1047 : static constexpr const char *EnumStr[] = {
1048 : "zeros", "ones", "lecun_normal",
1049 : "lecun_uniform", "xavier_normal", "xavier_uniform",
1050 : "he_normal", "he_uniform", "none"};
1051 : };
1052 :
1053 : /**
1054 : * @brief WeightInitializer Initialization Enumeration Information
1055 : *
1056 : */
1057 : class WeightInitializer final : public EnumProperty<InitializerInfo> {
1058 : public:
1059 : /**
1060 : * @brief Construct a WeightInitializer object
1061 : */
1062 : WeightInitializer(Initializer value = Initializer::XAVIER_UNIFORM);
1063 : using prop_tag = enum_class_prop_tag;
1064 : static constexpr const char *key = "weight_initializer";
1065 : };
1066 :
1067 : /**
1068 : * @brief BiasInitializer Initialization Enumeration Information
1069 : *
1070 : */
1071 : class BiasInitializer final : public EnumProperty<InitializerInfo> {
1072 : public:
1073 : /**
1074 : * @brief Construct a BiasInitializer object
1075 : */
1076 : BiasInitializer(Initializer value = Initializer::ZEROS);
1077 : using prop_tag = enum_class_prop_tag;
1078 : static constexpr const char *key = "bias_initializer";
1079 : };
1080 :
1081 : /**
1082 : * @brief MuInitializer Initialization Enumeration Information
1083 : *
1084 : */
1085 0 : class MuInitializer final : public EnumProperty<InitializerInfo> {
1086 : public:
1087 : /**
1088 : * @brief Construct a MuInitializer object
1089 : */
1090 : MuInitializer(Initializer value = Initializer::ZEROS);
1091 : using prop_tag = enum_class_prop_tag;
1092 : static constexpr const char *key = "moving_mean_initializer";
1093 : };
1094 :
1095 : /**
1096 : * @brief VarInitializer Initialization Enumeration Information
1097 : *
1098 : */
1099 0 : class VarInitializer final : public EnumProperty<InitializerInfo> {
1100 : public:
1101 : /**
1102 : * @brief Construct a VarInitializer object
1103 : */
1104 : VarInitializer(Initializer value = Initializer::ONES);
1105 : using prop_tag = enum_class_prop_tag;
1106 : static constexpr const char *key = "moving_variance_initializer";
1107 : };
1108 :
1109 : /**
1110 : * @brief GammaInitializer Initialization Enumeration Information
1111 : *
1112 : */
1113 189 : class GammaInitializer final : public EnumProperty<InitializerInfo> {
1114 : public:
1115 : /**
1116 : * @brief Construct a GammaInitializer object
1117 : */
1118 : GammaInitializer(Initializer value = Initializer::ONES);
1119 : using prop_tag = enum_class_prop_tag;
1120 : static constexpr const char *key = "gamma_initializer";
1121 : };
1122 :
1123 : /**
1124 : * @brief BetaInitializer Initialization Enumeration Information
1125 : *
1126 : */
1127 434 : class BetaInitializer final : public EnumProperty<InitializerInfo> {
1128 : public:
1129 : /**
1130 : * @brief Construct a BetaInitializer object
1131 : */
1132 : BetaInitializer(Initializer value = Initializer::ZEROS);
1133 : using prop_tag = enum_class_prop_tag;
1134 : static constexpr const char *key = "beta_initializer";
1135 : };
1136 :
1137 : /**
1138 : * @brief Enumeration of tensor regularization type
1139 : */
1140 : struct RegularizerInfo {
1141 : using Enum = nntrainer::WeightRegularizer;
1142 : static constexpr std::initializer_list<Enum> EnumList = {
1143 : Enum::L2NORM, Enum::NONE, Enum::UNKNOWN};
1144 :
1145 : static constexpr const char *EnumStr[] = {"l2norm", "none", "unknown"};
1146 : };
1147 :
1148 : /**
1149 : * @brief BasicRegularizer Regularization Enumeration Information
1150 : *
1151 : */
1152 0 : class BasicRegularizer : public EnumProperty<RegularizerInfo> {
1153 : public:
1154 : /**
1155 : * @brief Construct a BasicRegularizer object
1156 : */
1157 : BasicRegularizer(nntrainer::WeightRegularizer value);
1158 : using prop_tag = enum_class_prop_tag;
1159 : static constexpr const char *key = "basic_regularizer";
1160 :
1161 : /**
1162 : * @brief BasicRegularizer validator
1163 : *
1164 : * @param value nntrainer::WeightRegularizer to validate
1165 : * @retval true if value is not nntrainer::WeightRegularizer::UNKNOWN
1166 : * @retval false if value is nntrainer::WeightRegularizer::UNKNOWN
1167 : */
1168 : bool isValid(const nntrainer::WeightRegularizer &value) const override;
1169 : };
1170 :
1171 : /**
1172 : * @brief WeightRegularizer Regularization Enumeration Information
1173 : *
1174 : */
1175 : class WeightRegularizer final : public BasicRegularizer {
1176 : public:
1177 : /**
1178 : * @brief Construct a WeightRegularizer object
1179 : */
1180 : WeightRegularizer(
1181 : nntrainer::WeightRegularizer value = nntrainer::WeightRegularizer::NONE);
1182 : static constexpr const char *key = "weight_regularizer";
1183 : };
1184 :
1185 : /**
1186 : * @brief Enumeration of upsample type
1187 : * @todo Support torch and keras supported modes like bicubic
1188 : */
1189 : struct UpsampleModeInfo {
1190 : /**
1191 : * @brief Upsampling operation type class
1192 : */
1193 : enum class Interpolation { nearest, bilinear };
1194 :
1195 : using Enum = Interpolation;
1196 :
1197 : static constexpr std::initializer_list<Interpolation> EnumList = {
1198 : Interpolation::nearest, Interpolation::bilinear};
1199 :
1200 : static constexpr const char *EnumStr[] = {"nearest", "bilinear"};
1201 : };
1202 :
1203 : /**
1204 : * @brief Upsample Type Enumeration Information
1205 : *
1206 : */
1207 23 : class UpsampleMode final : public EnumProperty<UpsampleModeInfo> {
1208 : public:
1209 : using prop_tag = enum_class_prop_tag;
1210 : static constexpr const char *key = "upsample";
1211 : };
1212 :
1213 : /**
1214 : * @brief Enumeration of pooling type
1215 : */
1216 : struct PoolingTypeInfo {
1217 : /**
1218 : * @brief Pooling operation type class
1219 : */
1220 : enum class Enum {
1221 : max = 0,
1222 : average = 1,
1223 : global_max = 2,
1224 : global_average = 3,
1225 : unknown = 4
1226 : };
1227 : static constexpr std::initializer_list<Enum> EnumList = {
1228 : Enum::max, Enum::average, Enum::global_max, Enum::global_average,
1229 : Enum::unknown};
1230 :
1231 : static constexpr const char *EnumStr[] = {"max", "average", "global_max",
1232 : "global_average", "unknown"};
1233 : };
1234 :
1235 : /**
1236 : * @brief Pooling Type Enumeration Information
1237 : *
1238 : */
1239 131 : class PoolingType final : public EnumProperty<PoolingTypeInfo> {
1240 : public:
1241 : using prop_tag = enum_class_prop_tag;
1242 : static constexpr const char *key = "pooling";
1243 : };
1244 :
1245 : /**
1246 : * @brief Enumeration of flip direction
1247 : */
1248 : struct FlipDirectionInfo {
1249 : enum class Enum { horizontal, vertical, horizontal_and_vertical };
1250 : static constexpr std::initializer_list<Enum> EnumList = {
1251 : Enum::horizontal, Enum::vertical, Enum::horizontal_and_vertical};
1252 :
1253 : static constexpr const char *EnumStr[] = {"horizontal", "vertical",
1254 : "horizontal_and_vertical"};
1255 : };
1256 :
1257 : /**
1258 : * @brief FlipDirection Enumeration Information
1259 : *
1260 : */
1261 33 : class FlipDirection final : public EnumProperty<FlipDirectionInfo> {
1262 : public:
1263 : FlipDirection(FlipDirectionInfo::Enum value =
1264 : FlipDirectionInfo::Enum::horizontal_and_vertical);
1265 : using prop_tag = enum_class_prop_tag;
1266 : static constexpr const char *key = "flip_direction";
1267 : };
1268 :
1269 : /**
1270 : * @brief timestep property, timestep is used to identify for which timestep
1271 : * should the lstm/gru/rnn layer do the operation for
1272 : *
1273 : */
1274 540 : class Timestep : public Property<unsigned> {
1275 : public:
1276 : static constexpr const char *key = "timestep"; /**< unique key to access */
1277 : using prop_tag = uint_prop_tag; /**< property type */
1278 : };
1279 :
1280 : /**
1281 : * @brief maximum timestep property, timestep is used to identify for the
1282 : * maximum time unroll possible for lstm/gru/rnn layer
1283 : *
1284 : */
1285 986 : class MaxTimestep : public PositiveIntegerProperty {
1286 : public:
1287 : static constexpr const char *key =
1288 : "max_timestep"; /**< unique key to access */
1289 : using prop_tag = uint_prop_tag; /**< property type */
1290 : };
1291 :
1292 : /**
1293 : * @brief generic shape property which saves a single tensor shape
1294 : * (practically, std::array<GenericShape> is used)
1295 : *
1296 : * @note batch dimension is ignored with this dimension. Setting of batch must
1297 : * be done with the model.
1298 : *
1299 : */
1300 6 : class GenericShape : public Property<TensorDim> {
1301 :
1302 : public:
1303 : static constexpr const char *key =
1304 : "generic_shape"; /**< unique key to access */
1305 : using prop_tag = dimension_prop_tag; /**< property type */
1306 :
1307 : /**
1308 : * @brief Input shape setter
1309 : *
1310 : * @param value value to set
1311 : */
1312 : void set(const TensorDim &value) override;
1313 : };
1314 :
1315 : /**
1316 : * @brief target shape property which saves a single tensor shape
1317 : * (practically, std::array<TargetShape> is used)
1318 : *
1319 : */
1320 183 : class TargetShape : public GenericShape {
1321 :
1322 : public:
1323 : static constexpr const char *key =
1324 : "target_shape"; /**< unique key to access */
1325 : using prop_tag = dimension_prop_tag; /**< property type */
1326 : };
1327 :
1328 : /**
1329 : * @brief scaled dot product property, used to check
1330 : * whether attention layer is a kind of scaled dot product attention
1331 : *
1332 : */
1333 : class ScaledDotProduct : public nntrainer::Property<bool> {
1334 : public:
1335 : /**
1336 : * @brief Construct a new ScaledDotProduct object
1337 : *
1338 : */
1339 : ScaledDotProduct(bool value = false);
1340 : static constexpr const char *key = "scaled_dot_product";
1341 : using prop_tag = bool_prop_tag;
1342 : };
1343 :
1344 : /**
1345 : * @brief causal mask property, used in attention layer
1346 : *
1347 : */
1348 : class CausalMask : public nntrainer::Property<bool> {
1349 : public:
1350 : /**
1351 : * @brief Construct a new CausalMask object
1352 : *
1353 : */
1354 : CausalMask(bool value = false);
1355 : static constexpr const char *key = "causal_mask";
1356 : using prop_tag = bool_prop_tag;
1357 : };
1358 :
1359 : /**
1360 : * @brief Print object
1361 : *
1362 : */
1363 718 : class Print : public nntrainer::Property<bool> {
1364 : public:
1365 : /**
1366 : * @brief Construct a new Print object
1367 : *
1368 : */
1369 2804 : Print(bool value = false) { set(value); }
1370 : static constexpr const char *key = "print";
1371 : using prop_tag = bool_prop_tag;
1372 : };
1373 :
1374 : /**
1375 : * @brief K property, K is the size of the three projections in MoL attention
1376 : *
1377 : */
1378 19 : class MoL_K : public PositiveIntegerProperty {
1379 : public:
1380 : static constexpr const char *key = "MoL_K"; /**< unique key to access */
1381 : using prop_tag = uint_prop_tag; /**< property type */
1382 : };
1383 :
1384 : /**
1385 : * @brief NumHeads property, NumHeads is number of head in multi head attention
1386 : *
1387 : */
1388 298 : class NumHeads : public PositiveIntegerProperty {
1389 : public:
1390 : /**
1391 : * @brief Construct a new NumHeads object with default value 1
1392 : *
1393 : */
1394 : NumHeads(unsigned int value = 1);
1395 : static constexpr const char *key = "num_heads"; /**< unique key to access */
1396 : using prop_tag = uint_prop_tag; /**< property type */
1397 : };
1398 :
1399 : /**
1400 : * @brief ProjectedKeyDim property, projected key dim per head in multi head
1401 : * attention
1402 : * @details Correspond with key_dim of tensorflow
1403 : *
1404 : */
1405 149 : class ProjectedKeyDim : public PositiveIntegerProperty {
1406 : public:
1407 : static constexpr const char *key =
1408 : "projected_key_dim"; /**< unique key to access */
1409 : using prop_tag = uint_prop_tag; /**< property type */
1410 : };
1411 :
1412 : /**
1413 : * @brief ProjectedValueDim property, projected value dim per head in multi head
1414 : * attention
1415 : * @details Correspond with value_dim of tensorflow
1416 : *
1417 : */
1418 149 : class ProjectedValueDim : public PositiveIntegerProperty {
1419 : public:
1420 : static constexpr const char *key =
1421 : "projected_value_dim"; /**< unique key to access */
1422 : using prop_tag = uint_prop_tag; /**< property type */
1423 : };
1424 :
1425 : /**
1426 : * @brief OutputShape property, output shape of multi head
1427 : * attention
1428 : * @details Correspond with output_shape of tensorflow
1429 : *
1430 : */
1431 447 : class OutputShape : public PositiveIntegerProperty {
1432 : public:
1433 : static constexpr const char *key =
1434 : "output_shape"; /**< unique key to access */
1435 : using prop_tag = uint_prop_tag; /**< property type */
1436 : };
1437 :
1438 : /**
1439 : * @brief Enumeration of return attention weight
1440 : */
1441 : struct ReturnAttentionWeightInfo {
1442 : enum class Enum { none, before, after };
1443 : static constexpr std::initializer_list<Enum> EnumList = {
1444 : Enum::none, Enum::before, Enum::after};
1445 :
1446 : static constexpr const char *EnumStr[] = {"none", "before", "after"};
1447 : };
1448 :
1449 : /**
1450 : * @brief ReturnAttentionWeight, return attention weight
1451 : * @details "none" won't return attention weight.
1452 : * "before"/"after" will return attention weight before/after applying
1453 : * dropout
1454 : * @note Correspond with return_attention_scores of tensorflow and Correspond
1455 : * with need_weights of torch
1456 : *
1457 : */
1458 149 : class ReturnAttentionWeight : public EnumProperty<ReturnAttentionWeightInfo> {
1459 : public:
1460 : static constexpr const char *key =
1461 : "return_attention_weight"; /**< unique key to access */
1462 : using prop_tag = enum_class_prop_tag; /**< property type */
1463 :
1464 : /**
1465 : * @brief Construct a new ReturnAttentionWeight object
1466 : *
1467 : */
1468 : ReturnAttentionWeight(ReturnAttentionWeightInfo::Enum value =
1469 : ReturnAttentionWeightInfo::Enum::none);
1470 : };
1471 :
1472 : /**
1473 : * @brief AverageAttentionWeight, average attention weight
1474 : * @details Correspond with average_attn_weights of torch
1475 : *
1476 : */
1477 447 : class AverageAttentionWeight : public Property<bool> {
1478 : public:
1479 : static constexpr const char *key =
1480 : "average_attention_weight"; /**< unique key to access */
1481 : using prop_tag = bool_prop_tag; /**< property type */
1482 : };
1483 :
1484 : /**
1485 : * @brief LoRA rank property, it is used to set rank of LoRA weight.
1486 : * @details
1487 : */
1488 943 : class LoraRank : public PositiveIntegerProperty {
1489 : public:
1490 : static constexpr const char *key = "lora_rank"; /**< unique key to access */
1491 : using prop_tag = uint_prop_tag; /**< property type */
1492 : };
1493 :
1494 : /**
1495 : * @brief LoRA alpha parameter
1496 : * @details It is used to set the scaling factor of LoRA, which is calculated as
1497 : * `scaling = alpha / rank` in the original paper.
1498 : */
1499 1886 : class LoraAlpha : public PositiveIntegerProperty {
1500 : public:
1501 : static constexpr const char *key = "lora_alpha"; /**< unique key to access */
1502 : using prop_tag = uint_prop_tag; /**< property type */
1503 : };
1504 :
1505 : /**
1506 : * @brief properties for getting the clipping value to clip the gradient by norm
1507 : *
1508 : */
1509 21786 : class ClipGradByGlobalNorm : public Property<float> {
1510 : public:
1511 : static constexpr const char *key =
1512 : "clip_grad_by_norm"; /**< unique key to access */
1513 : using prop_tag = float_prop_tag; /**< property type */
1514 : };
1515 :
1516 : /**
1517 : * @brief properties for getting the loss scale value to mixed precision
1518 : *
1519 : */
1520 18573 : class LossScaleForMixed : public Property<float> {
1521 : public:
1522 : static constexpr const char *key = "loss_scale"; /**< unique key to access */
1523 : using prop_tag = float_prop_tag; /**< property type */
1524 :
1525 : /**
1526 : * @brief check if given value is valid
1527 : *
1528 : * @param value value to check
1529 : * @retval true if it is Not 0
1530 : * @retval false if it is 0
1531 : */
1532 : bool isValid(const float &value) const override;
1533 : };
1534 :
1535 : /**
1536 : * @brief Learning Rate props
1537 : *
1538 : */
1539 1485 : class LearningRate : public Property<float> {
1540 : public:
1541 : static constexpr const char *key =
1542 : "learning_rate"; /**< unique key to access */
1543 : using prop_tag = float_prop_tag; /**< property type */
1544 : };
1545 :
1546 : /**
1547 : * @brief Max Learning Rate props
1548 : *
1549 : */
1550 20 : class MaxLearningRate : public Property<float> {
1551 : public:
1552 : static constexpr const char *key =
1553 : "max_learning_rate"; /**< unique key to access */
1554 : using prop_tag = float_prop_tag; /**< property type */
1555 : };
1556 :
1557 : /**
1558 : * @brief Min Learning Rate props
1559 : *
1560 : */
1561 20 : class MinLearningRate : public Property<float> {
1562 : public:
1563 : static constexpr const char *key =
1564 : "min_learning_rate"; /**< unique key to access */
1565 : using prop_tag = float_prop_tag; /**< property type */
1566 : };
1567 :
1568 : /**
1569 : * @brief Iteration props
1570 : *
1571 : */
1572 51 : class Iteration : public Property<unsigned int> {
1573 : public:
1574 : static constexpr const char *key = "iteration"; /**< unique key to access */
1575 : using prop_tag = uint_prop_tag; /**< property type */
1576 : };
1577 :
1578 : /**
1579 : * @brief Decay rate property
1580 : *
1581 : */
1582 868 : class DecayRate : public Property<float> {
1583 : public:
1584 : static constexpr const char *key = "decay_rate"; /**< unique key to access */
1585 : using prop_tag = float_prop_tag; /**< property type */
1586 : };
1587 :
1588 : /**
1589 : * @brief decay steps property
1590 : *
1591 : */
1592 888 : class DecaySteps : public PositiveIntegerProperty {
1593 : public:
1594 : static constexpr const char *key = "decay_steps"; /**< unique key to access */
1595 : using prop_tag = uint_prop_tag; /**< property type */
1596 : };
1597 :
1598 : /**
1599 : * @brief User data props
1600 : *
1601 : */
1602 : class PropsUserData final : public Property<void *> {
1603 : public:
1604 : PropsUserData(void *user_data);
1605 : static constexpr const char *key = "user_data";
1606 : using prop_tag = ptr_prop_tag;
1607 : };
1608 :
1609 : /**
1610 : * @brief Enumeration of Tensor Life Spacn
1611 : */
1612 : struct TensorLifeInfo {
1613 : using Enum = nntrainer::TensorLifespan;
1614 : static constexpr std::initializer_list<Enum> EnumList = {
1615 : Enum::UNMANAGED,
1616 : Enum::FORWARD_FUNC_LIFESPAN,
1617 : Enum::CALC_DERIV_LIFESPAN,
1618 : Enum::CALC_GRAD_LIFESPAN,
1619 : Enum::CALC_AGRAD_LIFESPAN,
1620 : Enum::CALC_GRAD_DERIV_LIFESPAN,
1621 : Enum::CALC_GRAD_DERIV_AGRAD_LIFESPAN,
1622 : Enum::FORWARD_GRAD_LIFESPAN,
1623 : Enum::FORWARD_GRAD_AGRAD_LIFESPAN,
1624 : Enum::FORWARD_DERIV_LIFESPAN,
1625 : Enum::BACKWARD_FUNC_LIFESPAN,
1626 : Enum::CALC_GRAD_DERIV_AGRAD_LIFESPAN,
1627 : Enum::ITERATION_LIFESPAN,
1628 : Enum::EPOCH_LIFESPAN,
1629 : Enum::FORWARD_INFER_LIFESPAN,
1630 : Enum::MAX_LIFESPAN};
1631 :
1632 : static constexpr const char *EnumStr[] = {"unmanaged",
1633 : "forward",
1634 : "deriv",
1635 : "grad",
1636 : "agrad",
1637 : "grad_deriv",
1638 : "grad_deriv_agrad",
1639 : "forward_grad",
1640 : "forward_grad_agrad",
1641 : "forward_deriv",
1642 : "backward",
1643 : "grad_deriv_agrad",
1644 : "iteration",
1645 : "epoch",
1646 : "forward_infer",
1647 : "max"};
1648 : };
1649 :
1650 : /**
1651 : * @brief TensorLifeSpacn, return TensorLifespan
1652 : *
1653 : */
1654 0 : class TensorLife : public EnumProperty<TensorLifeInfo> {
1655 : public:
1656 : static constexpr const char *key = "tensor_life"; /**< unique key to access */
1657 : using prop_tag = enum_class_prop_tag; /**< property type */
1658 :
1659 : /**
1660 : * @brief Construct a new ReturnAttentionWeight object
1661 : *
1662 : */
1663 0 : TensorLife(TensorLifeInfo::Enum value = TensorLifeInfo::Enum::MAX_LIFESPAN) {
1664 0 : set(value);
1665 0 : };
1666 : };
1667 :
1668 : /**
1669 : * @brief WeightName, WeightName property
1670 : */
1671 111 : class WeightName : public Name {
1672 : public:
1673 : static constexpr const char *key = "weight_name";
1674 : using prop_tag = str_prop_tag;
1675 : };
1676 :
1677 : /**
1678 : * @brief TensorName, TensorName property
1679 : */
1680 0 : class TensorName : public Name {
1681 : public:
1682 : static constexpr const char *key = "tensor_name";
1683 : using prop_tag = str_prop_tag;
1684 : };
1685 :
1686 : } // namespace props
1687 : } // namespace nntrainer
1688 :
1689 : #endif // __COMMON_PROPERTIES_H__
|
