Initial import of libdpf.
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
commit
e4e666f459
4563 changed files with 1690372 additions and 0 deletions
51
include/grotto/gadgets/activations/celu.hpp
Normal file
51
include/grotto/gadgets/activations/celu.hpp
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
/// @file grotto/gadgets/activations/celu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_CELU_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_CELU_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double celu_default_alpha = 1;
|
||||
template <double const & alpha = celu_default_alpha>
|
||||
struct celu
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
return std::min(std::max(0, x) + std::min(0, alpha * std::expm1(x / alpha)), std::numeric_limits<T>::max());
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & alpha>
|
||||
struct gadget_hints<celu<alpha>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_CELU_HPP__
|
||||
51
include/grotto/gadgets/activations/elish.hpp
Normal file
51
include/grotto/gadgets/activations/elish.hpp
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
/// @file grotto/gadgets/activations/elish.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_ELISH_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_ELISH_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "sigmoid.hpp"
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct elish
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x < 0) return std::expm1(x)*sigmoid{}(x);
|
||||
if (x >= 0) return std::min(x*sigmoid{}(x), std::numeric_limits<T>::max());
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<elish>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_ELISH_HPP__
|
||||
52
include/grotto/gadgets/activations/elu.hpp
Normal file
52
include/grotto/gadgets/activations/elu.hpp
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
/// @file grotto/gadgets/activations/elu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_ELU_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_ELU_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double elu_default_alpha = 1;
|
||||
template <double const & alpha = elu_default_alpha>
|
||||
struct elu
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x <= 0) return alpha * std::expm1(x);
|
||||
if (x > 0) return std::min(x, std::numeric_limits<T>::max());
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & alpha>
|
||||
struct gadget_hints<elu<alpha>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_ELU_HPP__
|
||||
49
include/grotto/gadgets/activations/gelu.hpp
Normal file
49
include/grotto/gadgets/activations/gelu.hpp
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
/// @file grotto/gadgets/activations/gelu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_GELU_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_GELU_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct gelu
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
return std::min(x*(1+std::erf(x/std::sqrt(2)))/2, std::numeric_limits<T>::max());
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<gelu>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_GELU_HPP__
|
||||
50
include/grotto/gadgets/activations/hardelish.hpp
Normal file
50
include/grotto/gadgets/activations/hardelish.hpp
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/// @file grotto/gadgets/activations/hardelish.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDELISH_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDELISH_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct hardelish
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x < 0) return std::expm1(x)*std::max(0, std::min(1,(x+1)/2));
|
||||
if (x >= 0) return std::min(x*std::max(0,std::min(1,(x+1)/2)), std::numeric_limits<T>::max());
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<hardelish>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDELISH_HPP__
|
||||
52
include/grotto/gadgets/activations/hardshrink.hpp
Normal file
52
include/grotto/gadgets/activations/hardshrink.hpp
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
/// @file grotto/gadgets/activations/hardshrink.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDSHRINK_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDSHRINK_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double hardshrink_default_lambda = 0.5;
|
||||
template <double const & lambda = hardshrink_default_lambda>
|
||||
struct hardshrink
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (std::abs(x) <= lambda) return 0;
|
||||
return x;
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & lambda>
|
||||
struct gadget_hints<hardshrink<lambda>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { -lambda, lambda };
|
||||
static constexpr unsigned degree = 1;
|
||||
static constexpr bool has_canonical_representation = true;
|
||||
static constexpr double canonical_bounds[] = { -lambda, lambda };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { {0,1}, {0}, {0,1} };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDSHRINK_HPP__
|
||||
51
include/grotto/gadgets/activations/hardsigmoid.hpp
Normal file
51
include/grotto/gadgets/activations/hardsigmoid.hpp
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
/// @file grotto/gadgets/activations/hardsigmoid.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDSIGMOID_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDSIGMOID_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct hardsigmoid
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x <= -3) return 0;
|
||||
if (x >= 3) return 1;
|
||||
return (x+3)/6;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<hardsigmoid>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { -3, 3 };
|
||||
static constexpr unsigned degree = 1;
|
||||
static constexpr bool has_canonical_representation = true;
|
||||
static constexpr double canonical_bounds[] = { -3, 3 };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { {0}, {0.5,1/6.0}, {1} };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDSIGMOID_HPP__
|
||||
51
include/grotto/gadgets/activations/hardswish.hpp
Normal file
51
include/grotto/gadgets/activations/hardswish.hpp
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
/// @file grotto/gadgets/activations/hardswish.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDSWISH_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDSWISH_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct hardswish
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x <= -3) return 0;
|
||||
if (x >= 3) return x;
|
||||
return x*(x+3)/6;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<hardswish>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { -3, 3 };
|
||||
static constexpr unsigned degree = 2;
|
||||
static constexpr bool has_canonical_representation = true;
|
||||
static constexpr double canonical_bounds[] = { -3, 3 };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { {0}, {0,0.5,1/6.0}, {0,1,0} };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDSWISH_HPP__
|
||||
51
include/grotto/gadgets/activations/hardtanh.hpp
Normal file
51
include/grotto/gadgets/activations/hardtanh.hpp
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
/// @file grotto/gadgets/activations/hardtanh.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDTANH_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDTANH_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct hardtanh
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x <= -1) return -1;
|
||||
if (x >= 1) return 1;
|
||||
return x;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<hardtanh>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { -1, 1 };
|
||||
static constexpr unsigned degree = 1;
|
||||
static constexpr bool has_canonical_representation = true;
|
||||
static constexpr double canonical_bounds[] = { -1, 1 };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { {-1}, {0,1}, {1} };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_HARDTANH_HPP__
|
||||
53
include/grotto/gadgets/activations/leakyrelu.hpp
Normal file
53
include/grotto/gadgets/activations/leakyrelu.hpp
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/// @file grotto/gadgets/activations/leakyrelu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_LEAKYRELU_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_LEAKYRELU_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double leakyrelu_default_negative_slope = 0.01;
|
||||
static constexpr double leakyrelu_zero_negative_slope = 0.0;
|
||||
template <double const & negative_slope = leakyrelu_default_negative_slope>
|
||||
struct leakyrelu
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x >= 0) return x;
|
||||
return negative_slope * x;
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & negative_slope>
|
||||
struct gadget_hints<leakyrelu<negative_slope>>
|
||||
{
|
||||
inline static constexpr double min = std::numeric_limits<double>::lowest();
|
||||
inline static constexpr double max = std::numeric_limits<double>::max();
|
||||
inline static constexpr double poles[] = { };
|
||||
inline static constexpr double interesting_points[] = { 0 };
|
||||
inline static constexpr unsigned degree = 1;
|
||||
inline static constexpr bool has_canonical_representation = true;
|
||||
inline static constexpr double canonical_bounds[] = { 0 };
|
||||
inline static constexpr std::array<double, degree+1> canonical_polys[] = { {0,negative_slope}, {0,1} };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_LEAKYRELU_HPP__
|
||||
50
include/grotto/gadgets/activations/lecun_tanh.hpp
Normal file
50
include/grotto/gadgets/activations/lecun_tanh.hpp
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/// @file grotto/gadgets/activations/lecun_tanh.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_LECUNTANH_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_LECUNTANH_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct lecun_tanh
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
return 1.7159*std::tanh(2*x/3);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<lecun_tanh>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_LECUNTANH_HPP__
|
||||
47
include/grotto/gadgets/activations/logsigmoid.hpp
Normal file
47
include/grotto/gadgets/activations/logsigmoid.hpp
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
/// @file grotto/gadgets/activations/logsigmoid.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_LOGSIGMOID_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_LOGSIGMOID_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "sigmoid.hpp"
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct logsigmoid
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x) { return std::log(sigmoid{}(x)); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<logsigmoid>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_LOGSIGMOID_HPP__
|
||||
50
include/grotto/gadgets/activations/mish.hpp
Normal file
50
include/grotto/gadgets/activations/mish.hpp
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/// @file grotto/gadgets/activations/mish.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_MISH_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_MISH_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "softplus.hpp"
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct mish
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
return x*std::tanh(softplus<>{}(x));
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<mish>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_MISH_HPP__
|
||||
50
include/grotto/gadgets/activations/one_minus_sigmoid.hpp
Normal file
50
include/grotto/gadgets/activations/one_minus_sigmoid.hpp
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/// @file grotto/gadgets/activations/one_minus_relu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_ONE_MINUS_SIGMOID_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_ONE_MINUS_SIGMOID_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "sigmoid.hpp"
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct one_minus_sigmoid
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
return 1-sigmoid{}(x);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<one_minus_sigmoid>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_ONE_MINUS_SIGMOID_HPP__
|
||||
30
include/grotto/gadgets/activations/relu.hpp
Normal file
30
include/grotto/gadgets/activations/relu.hpp
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
/// @file grotto/gadgets/activations/relu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_RELU_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_RELU_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "leakyrelu.hpp"
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
using relu = leakyrelu<leakyrelu_zero_negative_slope>;
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_RELU_HPP__
|
||||
51
include/grotto/gadgets/activations/relu6.hpp
Normal file
51
include/grotto/gadgets/activations/relu6.hpp
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
/// @file grotto/gadgets/activations/relu6.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_RELU6_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_RELU6_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double relu6_default_clip = 6;
|
||||
template <double const & clip = 6>
|
||||
struct relu6
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
return std::min(std::max(0, x), clip);
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & clip>
|
||||
struct gadget_hints<relu6<clip>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { 0, clip };
|
||||
static constexpr unsigned degree = 1;
|
||||
static constexpr bool has_canonical_representation = true;
|
||||
static constexpr double canonical_bounds[] = { 0, clip };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { 0, {0,1}, clip };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_RELU6_HPP__
|
||||
53
include/grotto/gadgets/activations/selu.hpp
Normal file
53
include/grotto/gadgets/activations/selu.hpp
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/// @file grotto/gadgets/activations/selu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SELU_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SELU_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double selu_default_alpha = 1.6732632423543772848170429916717;
|
||||
static constexpr double selu_default_scale = 1.0507009873554804934193349852946;
|
||||
template <double const & alpha = selu_default_alpha,
|
||||
double const & scale = selu_default_scale>
|
||||
struct selu
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
return scale*(std::max(0,x)+std::min(0,alpha*std::expm1(x)));
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & alpha, double const & scale>
|
||||
struct gadget_hints<selu<alpha, scale>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SELU_HPP__
|
||||
50
include/grotto/gadgets/activations/serf.hpp
Normal file
50
include/grotto/gadgets/activations/serf.hpp
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/// @file grotto/gadgets/activations/serf.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SERF_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SERF_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "softplus.hpp"
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct serf
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
return x*std::erf(softplus<>{}(x));
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<serf>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SERF_HPP__
|
||||
46
include/grotto/gadgets/activations/sigmoid.hpp
Normal file
46
include/grotto/gadgets/activations/sigmoid.hpp
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
/// @file grotto/gadgets/activations/sigmoid.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SIGMOID_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SIGMOID_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct sigmoid
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x) { return 1/(1+std::exp(-x)); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<sigmoid>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SIGMOID_HPP__
|
||||
46
include/grotto/gadgets/activations/silu.hpp
Normal file
46
include/grotto/gadgets/activations/silu.hpp
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
/// @file grotto/gadgets/activations/silu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SILU_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SILU_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct silu
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x) { return x*sigmoid{}(x); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<silu>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SILU_HPP__
|
||||
53
include/grotto/gadgets/activations/smoothstep.hpp
Normal file
53
include/grotto/gadgets/activations/smoothstep.hpp
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/// @file grotto/gadgets/activations/smoothstep.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SMOOTHSTEP_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SMOOTHSTEP_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double smoothstep_default_gamma = 1;
|
||||
template <double const & gamma = smoothstep_default_gamma>
|
||||
struct smoothstep
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x <= -gamma/2) return 0;
|
||||
if (x <= gamma/2) return 1;
|
||||
if (std::abs(x) < gamma/2) return (-2/(gamma*gamma*gamma))*x*x*x+(3/(2*gamma))*x+0.5;
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & gamma>
|
||||
struct gadget_hints<smoothstep<gamma>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { -gamma/2, gamma/2 };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = true;
|
||||
static constexpr double canonical_bounds[] = { -gamma/2, gamma/2 };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { 0, {-2/(gamma*gamma*gamma),0,3/(2*gamma),0.5}, 1 };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SMOOTHSTEP_HPP__
|
||||
52
include/grotto/gadgets/activations/softminus.hpp
Normal file
52
include/grotto/gadgets/activations/softminus.hpp
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
/// @file grotto/gadgets/activations/softminus.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTMINUS_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTMINUS_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "softplus.hpp"
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double softminus_default_beta = 1;
|
||||
template <double const & beta = softminus_default_beta>
|
||||
struct softminus
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
return beta*x-softplus<beta>{}(x);
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & beta>
|
||||
struct gadget_hints<softminus<beta>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { 20 };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTMINUS_HPP__
|
||||
52
include/grotto/gadgets/activations/softplus.hpp
Normal file
52
include/grotto/gadgets/activations/softplus.hpp
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
/// @file grotto/gadgets/activations/softplus.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTPLUS_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTPLUS_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double softplus_default_beta = 1;
|
||||
template <double const & beta = softplus_default_beta>
|
||||
struct softplus
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x > 20) return x*beta;
|
||||
return (std::log1p(std::exp(beta*x)))/beta;
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & beta>
|
||||
struct gadget_hints<softplus<beta>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { 20 };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTPLUS_HPP__
|
||||
53
include/grotto/gadgets/activations/softshrink.hpp
Normal file
53
include/grotto/gadgets/activations/softshrink.hpp
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/// @file grotto/gadgets/activations/softshrink.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTSHRINK_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTSHRINK_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double softshrink_default_lambda = 0.5;
|
||||
template <double const & lambda = softshrink_default_lambda>
|
||||
struct softshrink
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (std::abs(x) <= lambda) return 0;
|
||||
if (x < lambda) return x+lambda;
|
||||
if (x > lambda) return x-lambda;
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & lambda>
|
||||
struct gadget_hints<softshrink<lambda>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { -lambda, lambda };
|
||||
static constexpr unsigned degree = 1;
|
||||
static constexpr bool has_canonical_representation = true;
|
||||
static constexpr double canonical_bounds[] = { -lambda, lambda };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { {lambda,1}, {0}, {-lambda,1} };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTSHRINK_HPP__
|
||||
46
include/grotto/gadgets/activations/softsign.hpp
Normal file
46
include/grotto/gadgets/activations/softsign.hpp
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
/// @file grotto/gadgets/activations/softsign.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTSIGN_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTSIGN_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct hardsign
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x) { return x/(1+std::abs(x)); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<hardsign>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 1;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SOFTSIGN_HPP__
|
||||
50
include/grotto/gadgets/activations/squared_relu.hpp
Normal file
50
include/grotto/gadgets/activations/squared_relu.hpp
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
/// @file grotto/gadgets/activations/squared_relu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SQUAREDRELU_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SQUAREDRELU_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct squared_relu
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x >= 0) return x*x;
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<squared_relu>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { 0 };
|
||||
static constexpr unsigned degree = 2;
|
||||
static constexpr bool has_canonical_representation = true;
|
||||
static constexpr double canonical_bounds[] = { 0 };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { {0}, {0,0,1} };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_SQUAREDRELU_HPP__
|
||||
54
include/grotto/gadgets/activations/starrelu.hpp
Normal file
54
include/grotto/gadgets/activations/starrelu.hpp
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
/// @file grotto/gadgets/activations/starrelu.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_STARRELU_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_STARRELU_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
static constexpr double starrelu_default_scale = 0.8944;
|
||||
static constexpr double starrelu_default_shift = -0.4472;
|
||||
template<double const & scale = starrelu_default_scale,
|
||||
double const & shift = starrelu_default_shift>
|
||||
struct starrelu
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x)
|
||||
{
|
||||
if (x < 0) return 0;
|
||||
return scale*x*x+shift;
|
||||
}
|
||||
};
|
||||
|
||||
template <double const & scale, double const & shift>
|
||||
struct gadget_hints<starrelu<shift, scale>>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { 0 };
|
||||
static constexpr unsigned degree = 2;
|
||||
static constexpr bool has_canonical_representation = true;
|
||||
static constexpr double canonical_bounds[] = { 0 };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { {0}, {0,0,1} };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_STARRELU_HPP__
|
||||
46
include/grotto/gadgets/activations/tanhexp.hpp
Normal file
46
include/grotto/gadgets/activations/tanhexp.hpp
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
/// @file grotto/gadgets/activations/tanhexp.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_TANHEXP_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_TANHEXP_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct tanhexp
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x) { return x*std::tanh(std::exp(x)); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<tanhexp>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_TANHEXP_HPP__
|
||||
46
include/grotto/gadgets/activations/tanhshrink.hpp
Normal file
46
include/grotto/gadgets/activations/tanhshrink.hpp
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
/// @file grotto/gadgets/activations/tanhshrink.hpp
|
||||
/// @author Ryan Henry <ryan.henry@ucalgary.ca>
|
||||
/// @brief
|
||||
/// @copyright Copyright (c) 2019-2023 Ryan Henry and others
|
||||
/// @license Released under a GNU General Public v2.0 (GPLv2) license;
|
||||
/// see [LICENSE.md](@ref GPLv2) for details.
|
||||
|
||||
#ifndef LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_TANHSHRINK_HPP__
|
||||
#define LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_TANHSHRINK_HPP__
|
||||
|
||||
#include <cmath>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
|
||||
#include "grotto/gadget_hints.hpp"
|
||||
|
||||
namespace grotto
|
||||
{
|
||||
|
||||
namespace gadgets
|
||||
{
|
||||
|
||||
struct tanhshrink
|
||||
{
|
||||
template <typename T>
|
||||
T operator()(T x) { return x -std::tanh(x); }
|
||||
};
|
||||
|
||||
template <>
|
||||
struct gadget_hints<tanhshrink>
|
||||
{
|
||||
static constexpr double min = std::numeric_limits<double>::min();
|
||||
static constexpr double max = std::numeric_limits<double>::max();
|
||||
static constexpr double poles[] = { };
|
||||
static constexpr double interesting_points[] = { };
|
||||
static constexpr unsigned degree = 3;
|
||||
static constexpr bool has_canonical_representation = false;
|
||||
static constexpr double canonical_bounds[] = { };
|
||||
static constexpr std::array<double, degree+1> canonical_polys[] = { };
|
||||
};
|
||||
|
||||
} // namespace gadgets
|
||||
|
||||
} // namespace grotto
|
||||
|
||||
#endif // LIBDPF_INCLUDE_GROTTO_GADGETS_ACTIVATIONS_TANHSHRINK_HPP__
|
||||
Loading…
Add table
Add a link
Reference in a new issue