Commit 2de9c3c8 authored by MattiaPujatti's avatar MattiaPujatti
Browse files

updated activations

parent 0c4d29a3
@phdthesis{Virtue:EECS-2019-126,
Author = {Virtue, Patrick},
author = {Virtue, Patrick},
Title = {Complex-valued Deep Learning with Applications to Magnetic Resonance Image Synthesis},
School = {EECS Department, University of California, Berkeley},
Year = {2019},
year = {2019},
Month = {Aug},
URL = {http://www2.eecs.berkeley.edu/Pubs/TechRpts/2019/EECS-2019-126.html},
Number = {UCB/EECS-2019-126},
......@@ -288,6 +288,7 @@
}
@book{hirose_cvnn,
author = {Hirose Akira},
title = {Complex-Valued Neural Networks: Advances and Applications},
publisher = {Wiley-IEEE Press},
isbn = {9781118344606; 111834460X; 9781118590072; 1118590074},
......@@ -343,3 +344,28 @@
archivePrefix={arXiv},
primaryClass={cs.CV}
}
@ARTICLE{Koutsougeras_siglog,
author={Georgiou, G.M. and Koutsougeras, C.},
journal={IEEE Transactions on Circuits and Systems II: Analog and Digital Signal Processing},
title={Complex domain backpropagation},
year={1992},
volume={39},
number={5},
pages={330-334},
doi={10.1109/82.142037}
}
@article{DBLP:journals/corr/ArjovskySB15,
author = {Martin Arjovsky and Amar Shah and Yoshua Bengio},
title = {Unitary Evolution Recurrent Neural Networks},
journal = {CoRR},
volume = {abs/1511.06464},
year = {2015},
url = {http://arxiv.org/abs/1511.06464},
eprinttype = {arXiv},
eprint = {1511.06464},
timestamp = {Mon, 13 Aug 2018 16:46:40 +0200},
biburl = {https://dblp.org/rec/journals/corr/ArjovskySB15.bib},
bibsource = {dblp computer science bibliography, https://dblp.org}
}
......@@ -63,5 +63,15 @@
\newlabel{eq:cmplx_batchnorm}{{1.3}{7}{Normalization Layers}{equation.1.3.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {1.4}Complex-Valued Activation Functions}{8}{section.1.4}\protected@file@percent }
\citation{Virtue:EECS-2019-126}
\citation{Nitta_complexBP}
\citation{Nitta_complexBP}
\citation{trabelsi2018deep}
\citation{guberman2016complex}
\citation{Koutsougeras_siglog}
\citation{Virtue:EECS-2019-126}
\citation{DBLP:journals/corr/ArjovskySB15}
\citation{Virtue:EECS-2019-126}
\@writefile{lot}{\contentsline {table}{\numberline {1.2}{\ignorespaces Recal of the most popular complex-valued activation functions.\relax }}{10}{table.caption.20}\protected@file@percent }
\newlabel{tab:cmplx_activations}{{1.2}{10}{Recal of the most popular complex-valued activation functions.\relax }{table.caption.20}{}}
\@writefile{toc}{\contentsline {section}{\numberline {1.5}JAX Implementation}{10}{section.1.5}\protected@file@percent }
\gdef \@abspage@last{10}
This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020/Debian) (preloaded format=pdflatex 2021.6.3) 5 NOV 2021 18:48
This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020/Debian) (preloaded format=pdflatex 2021.6.3) 6 NOV 2021 11:45
entering extended mode
restricted \write18 enabled.
%&-line parsing enabled.
......@@ -899,6 +899,38 @@ Package fancyhdr Warning: \headheight is too small (12.0pt):
[9]
LaTeX Warning: Citation `Nitta_complexBP' on page 10 undefined on input line 23
9.
LaTeX Warning: Citation `Nitta_complexBP' on page 10 undefined on input line 24
0.
LaTeX Warning: Citation `trabelsi2018deep' on page 10 undefined on input line 2
41.
LaTeX Warning: Citation `guberman2016complex' on page 10 undefined on input lin
e 242.
LaTeX Warning: Citation `Koutsougeras_siglog' on page 10 undefined on input lin
e 243.
LaTeX Warning: Citation `Virtue:EECS-2019-126' on page 10 undefined on input li
ne 244.
LaTeX Warning: Citation `DBLP:journals/corr/ArjovskySB15' on page 10 undefined
on input line 245.
LaTeX Warning: Citation `Virtue:EECS-2019-126' on page 10 undefined on input li
ne 246.
Package fancyhdr Warning: \headheight is too small (12.0pt):
(fancyhdr) Make it at least 13.59999pt, for example:
(fancyhdr) \setlength{\headheight}{13.59999pt}.
......@@ -914,10 +946,10 @@ Package rerunfilecheck Info: File `extent.out' has not changed.
(rerunfilecheck) Checksum: 69418383BC20A3C5ADE2D66D57B72767;594.
)
Here is how much of TeX's memory you used:
13015 strings out of 479304
193731 string characters out of 5869780
549901 words of memory out of 5000000
29880 multiletter control sequences out of 15000+600000
13019 strings out of 479304
193829 string characters out of 5869780
549997 words of memory out of 5000000
29884 multiletter control sequences out of 15000+600000
416756 words of font info for 81 fonts, out of 8000000 for 9000
1141 hyphenation exceptions out of 8191
116i,16n,120p,978b,414s stack positions out of 5000i,500n,10000p,200000b,80000s
......@@ -945,7 +977,7 @@ pfb></usr/share/texmf/fonts/type1/public/lm/lmss17.pfb></usr/share/texlive/texm
f-dist/fonts/type1/public/amsfonts/symbols/msam10.pfb></usr/share/texlive/texmf
-dist/fonts/type1/public/rsfs/rsfs10.pfb></usr/share/texmf/fonts/type1/public/c
m-super/sfrm1095.pfb>
Output written on extent.pdf (10 pages, 852441 bytes).
Output written on extent.pdf (10 pages, 853390 bytes).
PDF statistics:
266 PDF objects out of 1000 (max. 8388607)
216 compressed objects within 3 object streams
......
......@@ -199,7 +199,7 @@ This can be done easily with both the sigmoid and the hyperbolic tangent, mappin
Notice that this approach maps the phase of the signal into $[0,2\pi]$, since the function $g$ returns always a positive value.\\
There are also interesting variations to the separable sigmoid, properly designed to work using a complex-valued network on real-valued data. But, for this reason, they are functions with values in $\mathds{R}$ and not in $\mathds{C}$, and so we won't go through them in this work.\\
After the advent of the ReLU activation functions, two designs where developed in this fashion, the \texttt{$\mathds{C}$ReLU} and the \texttt{$z$ReLU}:
\[ \mathds{C}ReLU(z) = ReLU(\Re(z)) + iReLU(\Im(z)) \qquad\qquad zReLU = \begin{cases} z\quad \text{if } z\in[0,\pi/2] \\ 0\quad\text{otherwise} \end{cases} \]
\[ \mathds{C}ReLU(z) = ReLU(\Re(z)) + iReLU(\Im(z)) \qquad\qquad zReLU = \begin{cases} z\quad \text{if } \theta_z\in[0,\pi/2] \\ 0\quad\text{otherwise} \end{cases} \]
These functions also share the nice property of being holomorphic in some regions of the complex plane: \texttt{$\mathds{C}$ReLU} in the first and third quadrants, while \texttt{$z$ReLU} everywhere but the set of points $\left\{\Re(z)>0,\, \Im(z)=0\right\} \cup \left\{\Re(z)=0,\, \Im(z)>0\right\}$.
\subsection*{Phase-preserving Activations}
......@@ -209,7 +209,8 @@ It is called in this way because it is equivalent to applying the sigmoid to the
\[ siglog(z) = g\left(\log\norm{z}\right)e^{-i\angle z}= \frac{z}{1+\norm{z}}, \qquad\text{where}\quad g(x) = \frac{1}{1+e^{-x}} \]
Unlike the sigmoid and its separable version, the siglog projects the magnitude of the input from the interval $[0, \infty)$ to $[0,1)$. The authors of this proposal suggested also the addition of a couple of parameters to adjust the \textit{scale}, $r$, and the \textit{steepness}, $c$, of the function:
\[ siglog(z; r,c) = \frac{z}{c + \frac{1}{r}\norm{z}} \]
The main problem with \textit{siglog} is that the function has a nonzero gradient in the neighborhood of the origin of the complex plane, which can lead to gradient descent optimization algorithms to continuously stepping past the origin rather then approaching the point and staying here.\\
The main problem with \textit{siglog} is that the function has a nonzero gradient in the neighborhood of the origin of the complex plane, which can lead to gradient descent optimization algorithms
to continuously stepping past the origin rather then approaching the point and staying here.\\
For this reason, an alternative version have been proposed, this time with a better gradient behavior (approaching zero as the input approaches zero), that goes under the name of \texttt{iGaussian}.
\[ iGauss(z;\sigma^2)=g(z;\sigma^2)n(z) \qquad\text{where}\quad g(z;\sigma^2)=1-e^{-\frac{z\bar{z}}{2\sigma^2}},\quad n(z) = \frac{z}{\norm{z}} \]
This activation is basically an inverted gaussian (and this is the reason for which it is more smooth around the origin) and so depends only on one parameter, i.e. its standard deviation $\sigma$.\\
......@@ -233,12 +234,20 @@ We will see, in our applications, that the cardioid effectively allows complex n
\toprule
\textbf{Activation} & \textbf{Analytic Form} & \textbf{Reference}\\
\midrule
Sigmoid \\
Separable Sigmoid\\
Siglog\\
Sigmoid & $\sigma(z)$ & // \\
Hyperbolic Tangent & $\tanh(z)$ & // \\
Split-Sigmoid & $\sigma(\Re(z)) + i\sigma(\Im(z))$ & \cite{Nitta_complexBP}\\
Split-Tanh & $\tanh(\Re(z)) + i\tanh(\Im(z))$ & \cite{Nitta_complexBP}\\
$\mathds{C}$ReLU & $ReLU(\Re(z)) + iReLU(\Im(z))$ & \cite{trabelsi2018deep} \\
$z$ReLU & $z\;\text{if } z\in[0,\pi/2],\quad0\quad\text{otherwise}$ & \cite{guberman2016complex} \\
Siglog & $\sigma(\log\norm{z})e^{-i\theta_z}$ & \cite{Koutsougeras_siglog} \\
iGauss & $\left(1-e^{-\frac{z\bar{z}}{2\sigma^2}}\right)\frac{z}{\norm{z}}$ & \cite{Virtue:EECS-2019-126}\\
modReLU & $ReLU(\norm{z} + b)e^{i\theta_z}$ & \cite{DBLP:journals/corr/ArjovskySB15} \\
Cardioid & $\frac{1}{2}\left(1 + \cos(\angle z)\right)z$ & \cite{Virtue:EECS-2019-126}\\
\bottomrule
\end{tabular}
\caption{Recal of the most popular complex-valued activation functions.}
\label{tab:cmplx_activations}
\end{table}
\section{JAX Implementation}
......
......@@ -105,7 +105,16 @@
\newlabel{fig:cmplx_convolution}{{3.4}{xxiii}{Implementation details of the Complex Convolution (by \cite {trabelsi2018deep}).\relax }{figure.caption.19}{}}
\newlabel{eq:cmplx_batchnorm}{{3.3}{xxiii}{Normalization Layers}{equation.3.3.3}{}}
\@writefile{toc}{\contentsline {section}{\numberline {3.4}Complex-Valued Activation Functions}{xxiv}{section.3.4}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {3.5}JAX Implementation}{xxiv}{section.3.5}\protected@file@percent }
\citation{Virtue:EECS-2019-126}
\citation{Nitta_complexBP}
\citation{Nitta_complexBP}
\citation{trabelsi2018deep}
\citation{guberman2016complex}
\citation{Koutsougeras_siglog}
\citation{Virtue:EECS-2019-126}
\citation{DBLP:journals/corr/ArjovskySB15}
\citation{Virtue:EECS-2019-126}
\@writefile{toc}{\contentsline {section}{\numberline {3.5}JAX Implementation}{xxvi}{section.3.5}\protected@file@percent }
\citation{*}
\bibdata{bibliography}
\bibcite{trabelsi2018deep}{1}
......@@ -121,36 +130,38 @@
\bibcite{xavier_init}{11}
\bibcite{he2015delving}{12}
\bibcite{cogswell2016reducing}{13}
\bibcite{ziller2021complexvalued}{14}
\bibcite{article}{15}
\bibcite{4682548}{16}
\bibcite{virtue2017better}{17}
\bibcite{guberman2016complex}{18}
\bibcite{GARDNER2021116245}{19}
\bibcite{ajakan2015domainadversarial}{20}
\bibcite{ganin2016domainadversarial}{21}
\bibcite{reichert2014neuronal}{22}
\bibcite{scardapane2018complexvalued}{23}
\bibcite{barrachina2021complexvalued}{24}
\bibcite{shen2018wasserstein}{25}
\bibcite{schlomer_nico_2021_5636188}{26}
\bibcite{Farris_visual_complex_analysis}{27}
\bibcite{guberman2016complex}{14}
\bibcite{Koutsougeras_siglog}{15}
\bibcite{DBLP:journals/corr/ArjovskySB15}{16}
\bibcite{ziller2021complexvalued}{17}
\bibcite{article}{18}
\bibcite{4682548}{19}
\bibcite{virtue2017better}{20}
\bibcite{GARDNER2021116245}{21}
\bibcite{ajakan2015domainadversarial}{22}
\bibcite{ganin2016domainadversarial}{23}
\bibcite{reichert2014neuronal}{24}
\bibcite{scardapane2018complexvalued}{25}
\bibcite{barrachina2021complexvalued}{26}
\bibcite{shen2018wasserstein}{27}
\bibcite{schlomer_nico_2021_5636188}{28}
\bibcite{Farris_visual_complex_analysis}{29}
\bibstyle{ieeetr}
\citation{trabelsi2018deep}
\citation{xavier_init}
\citation{he2015delving}
\citation{xavier_init}
\citation{he2015delving}
\@writefile{toc}{\contentsline {chapter}{\numberline {A}Mathematical Proofs}{xxvii}{appendix.A}\protected@file@percent }
\@writefile{toc}{\contentsline {chapter}{\numberline {A}Mathematical Proofs}{xxix}{appendix.A}\protected@file@percent }
\@writefile{lof}{\addvspace {10\p@ }}
\@writefile{lot}{\addvspace {10\p@ }}
\newlabel{app:cmplx_optim}{{A}{xxvii}{Mathematical Proofs}{appendix.A}{}}
\@writefile{toc}{\contentsline {section}{\numberline {A.1}Complex Weights Initialization \cite {trabelsi2018deep}}{xxvii}{section.A.1}\protected@file@percent }
\newlabel{app:weight_init}{{A.1}{xxvii}{Complex Weights Initialization \cite {trabelsi2018deep}}{section.A.1}{}}
\newlabel{app:cmplx_optim}{{A}{xxix}{Mathematical Proofs}{appendix.A}{}}
\@writefile{toc}{\contentsline {section}{\numberline {A.1}Complex Weights Initialization \cite {trabelsi2018deep}}{xxix}{section.A.1}\protected@file@percent }
\newlabel{app:weight_init}{{A.1}{xxix}{Complex Weights Initialization \cite {trabelsi2018deep}}{section.A.1}{}}
\citation{Messerschmitt_stationary_points}
\citation{MESSERSCHMITT_STATIONARY_POINTS}
\@writefile{toc}{\contentsline {section}{\numberline {A.2}Stationary points of a real-valued function of a complex variable \cite {Messerschmitt_stationary_points}}{xxviii}{section.A.2}\protected@file@percent }
\@writefile{toc}{\contentsline {section}{\numberline {A.2}Stationary points of a real-valued function of a complex variable \cite {Messerschmitt_stationary_points}}{xxx}{section.A.2}\protected@file@percent }
\citation{Hualiang_nonlinear}
\citation{HUALIANG_NONLINEAR}
\@writefile{toc}{\contentsline {section}{\numberline {A.3}Steepest complex gradient descent \cite {Hualiang_nonlinear}}{xxix}{section.A.3}\protected@file@percent }
\gdef \@abspage@last{29}
\@writefile{toc}{\contentsline {section}{\numberline {A.3}Steepest complex gradient descent \cite {Hualiang_nonlinear}}{xxxi}{section.A.3}\protected@file@percent }
\gdef \@abspage@last{31}
......@@ -38,7 +38,7 @@ P.~Virtue, {\em Complex-valued Deep Learning with Applications to Magnetic
2019.
\bibitem{hirose_cvnn}
{\em Complex-Valued Neural Networks: Advances and Applications}.
H.~Akira, {\em Complex-Valued Neural Networks: Advances and Applications}.
\newblock Wiley-IEEE Press, 2013.
\bibitem{amin_wirtinger}
......@@ -59,6 +59,19 @@ K.~He, X.~Zhang, S.~Ren, and J.~Sun, ``Delving deep into rectifiers: Surpassing
M.~Cogswell, F.~Ahmed, R.~Girshick, L.~Zitnick, and D.~Batra, ``Reducing
overfitting in deep networks by decorrelating representations,'' 2016.
\bibitem{guberman2016complex}
N.~Guberman, ``On complex valued convolutional neural networks.''
\url{https://arxiv.org/abs/1602.09046}, 2016.
\bibitem{Koutsougeras_siglog}
G.~Georgiou and C.~Koutsougeras, ``Complex domain backpropagation,'' {\em IEEE
Transactions on Circuits and Systems II: Analog and Digital Signal
Processing}, vol.~39, no.~5, pp.~330--334, 1992.
\bibitem{DBLP:journals/corr/ArjovskySB15}
M.~Arjovsky, A.~Shah, and Y.~Bengio, ``Unitary evolution recurrent neural
networks,'' {\em CoRR}, vol.~abs/1511.06464, 2015.
\bibitem{ziller2021complexvalued}
A.~Ziller, D.~Usynin, M.~Knolle, K.~Hammernik, D.~Rueckert, and G.~Kaissis,
``Complex-valued deep learning with differential privacy.''
......@@ -77,10 +90,6 @@ E.~Ollila, ``On the circularity of a complex random variable,'' {\em IEEE
P.~Virtue, S.~X. Yu, and M.~Lustig, ``Better than real: Complex-valued neural
nets for mri fingerprinting.'' \url{https://arxiv.org/abs/1707.00070}, 2017.
\bibitem{guberman2016complex}
N.~Guberman, ``On complex valued convolutional neural networks.''
\url{https://arxiv.org/abs/1602.09046}, 2016.
\bibitem{GARDNER2021116245}
P.~Gardner, L.~Bull, N.~Dervilis, and K.~Worden, ``Overcoming the problem of
repair in structural health monitoring: Metric-informed transfer learning,''
......
......@@ -3,12 +3,12 @@ Capacity: max_strings=200000, hash_size=200000, hash_prime=170003
The top-level auxiliary file: main.aux
The style file: ieeetr.bst
Case mismatch error between cite keys MESSERSCHMITT_STATIONARY_POINTS and Messerschmitt_stationary_points
---line 149 of file main.aux
---line 133 of file main.aux
: \citation{MESSERSCHMITT_STATIONARY_POINTS
: }
I'm skipping whatever remains of this command
Case mismatch error between cite keys HUALIANG_NONLINEAR and Hualiang_nonlinear
---line 152 of file main.aux
---line 136 of file main.aux
: \citation{HUALIANG_NONLINEAR
: }
I'm skipping whatever remains of this command
......@@ -37,47 +37,46 @@ I was expecting a `,' or a `}'---line 254 of file bibliography.bib
(Error may have been on previous line)
I'm skipping whatever remains of this entry
Warning--empty journal in Dramsch_seismic
Warning--empty author and editor in hirose_cvnn
Warning--empty booktitle in amin_wirtinger
You've used 27 entries,
You've used 29 entries,
1876 wiz_defined-function locations,
634 strings with 8893 characters,
and the built_in function-call counts, 5250 in all, are:
= -- 444
> -- 305
649 strings with 9273 characters,
and the built_in function-call counts, 5755 in all, are:
= -- 493
> -- 327
< -- 1
+ -- 114
- -- 87
* -- 368
:= -- 806
add.period$ -- 31
call.type$ -- 27
change.case$ -- 24
+ -- 122
- -- 93
* -- 402
:= -- 884
add.period$ -- 33
call.type$ -- 29
change.case$ -- 26
chr.to.int$ -- 0
cite$ -- 30
duplicate$ -- 218
empty$ -- 522
format.name$ -- 87
if$ -- 1259
cite$ -- 31
duplicate$ -- 243
empty$ -- 569
format.name$ -- 93
if$ -- 1383
int.to.chr$ -- 0
int.to.str$ -- 27
missing$ -- 12
newline$ -- 88
num.names$ -- 26
pop$ -- 125
int.to.str$ -- 29
missing$ -- 15
newline$ -- 94
num.names$ -- 29
pop$ -- 129
preamble$ -- 1
purify$ -- 0
quote$ -- 0
skip$ -- 161
skip$ -- 177
stack$ -- 0
substring$ -- 120
swap$ -- 63
substring$ -- 148
swap$ -- 73
text.length$ -- 1
text.prefix$ -- 0
top$ -- 0
type$ -- 0
warning$ -- 3
while$ -- 36
width$ -- 29
write$ -- 235
warning$ -- 2
while$ -- 41
width$ -- 31
write$ -- 256
(There were 7 error messages)
This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020/Debian) (preloaded format=pdflatex 2021.6.3) 5 NOV 2021 11:24
This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020/Debian) (preloaded format=pdflatex 2021.6.3) 6 NOV 2021 11:08
entering extended mode
restricted \write18 enabled.
%&-line parsing enabled.
......@@ -682,22 +682,22 @@ Overfull \hbox (3.40668pt too wide) detected at line 20
Overfull \hbox (3.40668pt too wide) detected at line 21
[]\OT1/cmr/m/n/10.95 xxiv
[]\OT1/cmr/m/n/10.95 xxvi
[]
Overfull \hbox (6.44835pt too wide) detected at line 23
[]\OT1/cmr/m/n/10.95 xxvii
Overfull \hbox (3.40668pt too wide) detected at line 23
[]\OT1/cmr/m/n/10.95 xxix
[]
Overfull \hbox (9.49002pt too wide) detected at line 24
[]\OT1/cmr/m/n/10.95 xxviii
Overfull \hbox (0.365pt too wide) detected at line 24
[]\OT1/cmr/m/n/10.95 xxx
[]
Overfull \hbox (3.40668pt too wide) detected at line 25
[]\OT1/cmr/m/n/10.95 xxix
[]\OT1/cmr/m/n/10.95 xxxi
[]
)
......@@ -916,11 +916,20 @@ Package fancyhdr Warning: \headheight is too small (12.0pt):
(fancyhdr) \addtolength{\topmargin}{-1.59999pt}.
[23 <./pictures/complex_convolution.pdf>]
Underfull \hbox (badness 10000) in paragraph at lines 186--190
Package fancyhdr Warning: \headheight is too small (12.0pt):
(fancyhdr) Make it at least 13.59999pt, for example:
(fancyhdr) \setlength{\headheight}{13.59999pt}.
(fancyhdr) You might also make \topmargin smaller to compensate:
(fancyhdr) \addtolength{\topmargin}{-1.59999pt}.
[24]
Overfull \hbox (2.51306pt too wide) in paragraph at lines 222--225
\OT1/cmr/m/n/10.95 Because of this, re-cently a new com-plex ac-ti-va-tion func
-tion have been pro-posed: the \OT1/cmtt/m/n/10.95 Complex Cardioid
[]
)
Package fancyhdr Warning: \headheight is too small (12.0pt):
(fancyhdr) Make it at least 13.59999pt, for example:
......@@ -929,7 +938,16 @@ Package fancyhdr Warning: \headheight is too small (12.0pt):
(fancyhdr) \addtolength{\topmargin}{-1.59999pt}.
[24] (./main.bbl [25
[25])
Package fancyhdr Warning: \headheight is too small (12.0pt):
(fancyhdr) Make it at least 13.59999pt, for example:
(fancyhdr) \setlength{\headheight}{13.59999pt}.
(fancyhdr) You might also make \topmargin smaller to compensate:
(fancyhdr) \addtolength{\topmargin}{-1.59999pt}.
[26] (./main.bbl [27
])
......@@ -940,7 +958,7 @@ Package fancyhdr Warning: \headheight is too small (12.0pt):
(fancyhdr) \addtolength{\topmargin}{-1.59999pt}.
[26] (./chapters/appendix.tex
[28] (./chapters/appendix.tex
Appendix A.
Package hyperref Warning: Token not allowed in a PDF string (PDFDocEncoding):
......@@ -951,7 +969,7 @@ Underfull \hbox (badness 10000) in paragraph at lines 25--28
[]
[27
[29
]
......@@ -982,8 +1000,8 @@ Underfull \hbox (badness 10000) in paragraph at lines 54--58
[]
LaTeX Warning: Citation `MESSERSCHMITT_STATIONARY_POINTS' on page xxviii undefi
ned on input line 62.
LaTeX Warning: Citation `MESSERSCHMITT_STATIONARY_POINTS' on page xxx undefined
on input line 62.
Package fancyhdr Warning: \headheight is too small (12.0pt):
......@@ -993,7 +1011,7 @@ Package fancyhdr Warning: \headheight is too small (12.0pt):
(fancyhdr) \addtolength{\topmargin}{-13.16252pt}.
[28]
[30]
Package hyperref Warning: Token not allowed in a PDF string (PDFDocEncoding):
(hyperref) removing `\@ifnextchar' on input line 68.
......@@ -1016,8 +1034,8 @@ n/10.95 r[] \OT1/cmr/m/n/10.95 = []$ for the vec-tor $\OT1/cmr/bx/n/10.95 z \OT
)
LaTeX Warning: Citation `HUALIANG_NONLINEAR' on page xxix undefined on input li
ne 55.
LaTeX Warning: Citation `HUALIANG_NONLINEAR' on page xxxi undefined on input li
ne 54.
Package fancyhdr Warning: \headheight is too small (12.0pt):
......@@ -1027,7 +1045,7 @@ Package fancyhdr Warning: \headheight is too small (12.0pt):
(fancyhdr) \addtolength{\topmargin}{-1.59999pt}.
[29] (./main.aux)
[31] (./main.aux)
LaTeX Warning: There were undefined references.
......@@ -1035,10 +1053,10 @@ Package rerunfilecheck Info: File `main.out' has not changed.
(rerunfilecheck) Checksum: 0BA567321A3D0FA80C405B5F82C8F9FE;1767.
)
Here is how much of TeX's memory you used:
13199 strings out of 479304
197237 string characters out of 5869780
551956 words of memory out of 5000000
29955 multiletter control sequences out of 15000+600000
13211 strings out of 479304
197439 string characters out of 5869780
552050 words of memory out of 5000000
29957 multiletter control sequences out of 15000+600000
418067 words of font info for 85 fonts, out of 8000000 for 9000
1141 hyphenation exceptions out of 8191
96i,16n,103p,1169b,537s stack positions out of 5000i,500n,10000p,200000b,80000s
......@@ -1068,10 +1086,10 @@ r/share/texmf/fonts/type1/public/lm/lmss17.pfb></usr/share/texlive/texmf-dist/f
onts/type1/public/amsfonts/symbols/msam10.pfb></usr/share/texlive/texmf-dist/fo
nts/type1/public/rsfs/rsfs10.pfb></usr/share/texmf/fonts/type1/public/cm-super/
sfrm1095.pfb>
Output written on main.pdf (29 pages, 3796885 bytes).
Output written on main.pdf (31 pages, 3809813 bytes).
PDF statistics:
599 PDF objects out of 1000 (max. 8388607)
512 compressed objects within 6 object streams
123 named destinations out of 1000 (max. 500000)
627 PDF objects out of 1000 (max. 8388607)
538 compressed objects within 6 object streams
133 named destinations out of 1000 (max. 500000)
228 words of extra memory for PDF output out of 10000 (max. 10000000)
No preview for this file type
No preview for this file type
......@@ -30,7 +30,6 @@
\subfile{chapters/extent}
%------------------------------------------------
%------------------------------------------------
%\subfile{sections/theory.tex}
%------------------------------------------------
......
......@@ -18,8 +18,8 @@
\contentsline {subsection}{\numberline {3.2.2}Backpropagation with a Real-valued Loss}{xx}{subsection.3.2.2}%
\contentsline {section}{\numberline {3.3}Re-definition of the main neural network layers}{xxi}{section.3.3}%
\contentsline {section}{\numberline {3.4}Complex-Valued Activation Functions}{xxiv}{section.3.4}%
\contentsline {section}{\numberline {3.5}JAX Implementation}{xxiv}{section.3.5}%
\contentsline {chapter}{\numberline {A}Mathematical Proofs}{xxvii}{appendix.A}%
\contentsline {section}{\numberline {A.1}Complex Weights Initialization \cite {trabelsi2018deep}}{xxvii}{section.A.1}%
\contentsline {section}{\numberline {A.2}Stationary points of a real-valued function of a complex variable \cite {Messerschmitt_stationary_points}}{xxviii}{section.A.2}%
\contentsline {section}{\numberline {A.3}Steepest complex gradient descent \cite {Hualiang_nonlinear}}{xxix}{section.A.3}%
\contentsline {section}{\numberline {3.5}JAX Implementation}{xxvi}{section.3.5}%
\contentsline {chapter}{\numberline {A}Mathematical Proofs}{xxix}{appendix.A}%
\contentsline {section}{\numberline {A.1}Complex Weights Initialization \cite {trabelsi2018deep}}{xxix}{section.A.1}%
\contentsline {section}{\numberline {A.2}Stationary points of a real-valued function of a complex variable \cite {Messerschmitt_stationary_points}}{xxx}{section.A.2}%
\contentsline {section}{\numberline {A.3}Steepest complex gradient descent \cite {Hualiang_nonlinear}}{xxxi}{section.A.3}%
......@@ -57,3 +57,4 @@
\theoremstyle{definition}
\newtheorem{observation}{Observation}[section]
This is pdfTeX, Version 3.14159265-2.6-1.40.21 (TeX Live 2020/Debian) (preloaded format=pdflatex 2021.6.3) 27 OCT 2021 10:08
entering extended mode
restricted \write18 enabled.
%&-line parsing enabled.
**preamble.tex
! Emergency stop.
<*> preamble.tex
*** (job aborted, file error in nonstop mode)
Here is how much of TeX's memory you used:
3 strings out of 479304
110 string characters out of 5869780
271124 words of memory out of 5000000
17297 multiletter control sequences out of 15000+600000
403430 words of font info for 27 fonts, out of 8000000 for 9000
1141 hyphenation exceptions out of 8191
0i,0n,0p,1b,6s stack positions out of 5000i,500n,10000p,200000b,80000s
! ==> Fatal error occurred, no output PDF file produced!
""" Definition of some complex activation functions according to the literature. """
import jax
import jax.numpy as jnp
import numpy as np
import torch
from jax import lax
from jax import custom_jvp
from jax import lax, custom_jvp
def cmplx_sigmoid(z):
"""Implementation of the complex sigmoid."""
return 1. / ( 1. + jnp.exp(-z) )
def cmplx_tanh(z):
"""Implementation of the complex hyperbolic tangent."""
return jnp.tanh(z)