Resumen: Como mi post es largo, esto es lo que finalmente probaré:
\begin{equation} \sum_{j=1}^{n}{(-1)^jexp \left( \frac{2 \pi i j^2k}{2n} \right)}= \begin{cases} -1 & \text{if $n$ is odd and $k$ is even} \\ \sqrt{n}exp\left(i \pi \frac{1-n}{4}\right)\left( \frac{k}{n}\right) & \text{if $n$ is odd and $k$ is odd} \\ \sqrt{n}exp\left(i \pi \frac{2-k-n}{4}\right)\left( \frac{k}{n+k}\right) & \text{if $n$ is even (so $k$ is odd)} \end{cases} \end{equation}
Argumento principal: Parece que sabes cómo obtener lo siguiente utilizando las sumas de Gauss:
Si $n+k$ es impar (así que $\gcd(n+k, 2n)=1$ ): \begin{equation} \sum_{j=0}^{2n-1}{exp \left( \frac{2 \pi i j^2(n+k)}{2n} \right)}= \begin{cases} 0 & \text{if $n$ is odd} \\ (1+i)\epsilon_{n+k}^{-1}\sqrt{2n} \left(\frac{2n}{n+k}\right) & \text{if $n$ is even } \end{cases} \end{equation}
Si $n+k$ es par (por lo que $n$ es impar, ya que $\gcd(n, k)=1$ ) \begin{equation} \sum_{j=1}^{n}{exp \left( \frac{2 \pi i j^2\frac{(n+k)}{2}}{n} \right)}=\sum_{j=0}^{n-1}{exp \left( \frac{2 \pi i j^2\frac{(n+k)}{2}}{n} \right)}=\epsilon_n \sqrt{n} \left( \frac{\frac{n+k}{2}}{n}\right) \end{equation} donde \begin{equation} \epsilon_{m}= \begin{cases} 1 & \text{if} \; m \equiv 1 \pmod 4 \\ i & \text{if} \; m \equiv 3 \pmod 4 \end{cases} \end{equation} para cualquier entero impar $m$ et $\left( \frac{\frac{n+k}{2}}{n}\right)$ y $\left(\frac{2n}{n+k}\right)$ se refieren al símbolo de Jacobi.
Ahora,
\begin{align} \sum_{j=1}^{n}{(-1)^jexp \left( \frac{2 \pi i j^2k}{2n} \right)}& =\sum_{j=1}^{n}{(-1)^{j^2}exp \left( \frac{2 \pi i j^2k}{2n} \right)} \\ & =\sum_{j=1}^{n}{exp \left( \frac{2 \pi i j^2(n+k)}{2n} \right)} \\ & =\frac{1}{2}\left(exp \left( \frac{2 \pi i n^2(n+k)}{2n} \right)-1+\sum_{j=0}^{2n-1}{exp \left( \frac{2 \pi i j^2(n+k)}{2n} \right)}\right) \\ & =\frac{1}{2}\left((-1)^{n(n+k)}-1+\sum_{j=0}^{2n-1}{exp \left( \frac{2 \pi i j^2(n+k)}{2n} \right)}\right) \end{align}
Así,
\begin{equation} \sum_{j=1}^{n}{(-1)^jexp \left( \frac{2 \pi i j^2k}{2n} \right)}= \begin{cases} -1 & \text{if $n$ is odd and $k$ is even} \\ \epsilon_n \sqrt{n} \left( \frac{\frac{n+k}{2}}{n}\right) & \text{if $n$ is odd and $k$ is odd} \\ (1+i)\epsilon_{n+k}^{-1}\sqrt{\frac{n}{2}} \left(\frac{2n}{n+k}\right) & \text{if $n$ is even (so $k$ is odd)} \end{cases} \end{equation}
Esta fórmula ya es agradable. Voy a manipular para dar una expresión diferente (más simple).
$n, k$ son ambos Impares: \begin{equation} \left( \frac{\frac{n+k}{2}}{n}\right)=\left( \frac{\frac{n+k}{2}}{n}\right)\left( \frac{2}{n}\right)\left( \frac{2}{n}\right)=\left( \frac{n+k}{n}\right)\left( \frac{2}{n}\right)=\left( \frac{2}{n}\right)\left( \frac{k}{n}\right)=(-1)^{\frac{n^2-1}{8}}\left( \frac{k}{n}\right) \end{equation}
Si $n \equiv 1 \pmod 4$ entonces $\epsilon_n=1$ y $$\epsilon_n(-1)^{\frac{n^2-1}{8}}=(-1)^{\frac{n^2-1}{8}}=(-1)^{\frac{n-1}{4}}=(-1)^{\frac{1-n}{4}}=exp\left(i \pi \frac{1-n}{4}\right)$$
Si $n \equiv 3 \pmod 4$ entonces $\epsilon_n=i$ y $$\epsilon_n(-1)^{\frac{n^2-1}{8}}=i(-1)^{\frac{n^2-1}{8}}=i(-1)^{\frac{n+1}{4}}=i(-1)^{-\frac{n+1}{4}}=exp\left(i \pi \left(\frac{1}{2}-\frac{n+1}{4}\right)\right)=exp\left(i \pi \frac{1-n}{4}\right)$$
Así, para el caso en que $n, k$ tanto impar, tenemos $$\sum_{j=1}^{n}{(-1)^jexp \left( \frac{2 \pi i j^2k}{2n} \right)}=\sqrt{n}exp\left(i \pi \frac{1-n}{4}\right)\left( \frac{k}{n}\right)$$
$n$ es incluso entonces $k$ es impar, así que \begin{equation} \left(\frac{2n}{n+k}\right)=\left(\frac{-2k}{n+k}\right)=\left(\frac{-1}{n+k}\right)\left(\frac{2}{n+k}\right)\left(\frac{k}{n+k}\right)=(-1)^{\frac{(n+k)-1}{2}}(-1)^{\frac{(n+k)^2-1}{8}}\left(\frac{k}{n+k}\right) \end{equation}
Si $n+k \equiv 1 \pmod 4$ tenemos $\epsilon_{n+k}=1$ , \begin{equation} (-1)^{\frac{(n+k)-1}{2}}(-1)^{\frac{(n+k)^2-1}{8}}\left(\frac{k}{n+k}\right)=(-1)^{\frac{(n+k)-1}{4}}\left(\frac{k}{n+k}\right)=(-1)^{\frac{1-(n+k)}{4}}\left(\frac{k}{n+k}\right) \end{equation}
Así, \begin{align} \frac{1+i}{\sqrt{2}}\epsilon_{n+k}^{-1}\left(\frac{2n}{n+k}\right) & =exp \left( i \pi \frac{1}{4}\right)exp \left( i \pi \frac{1-(n+k)}{4}\right)\left(\frac{k}{n+k}\right) \\ & =exp \left( i \pi \frac{2-k-n}{4}\right)\left(\frac{k}{n+k}\right) \end{align}
Si $n+k \equiv 3 \pmod 4$ tenemos $\epsilon_{n+k}=i$ , \begin{align} (-1)^{\frac{(n+k)-1}{2}}(-1)^{\frac{(n+k)^2-1}{8}}\left(\frac{k}{n+k}\right)& =(-1)(-1)^{\frac{(n+k)+1}{4}}\left(\frac{k}{n+k}\right) \\ & =(-1)(-1)^{-\frac{(n+k)+1}{4}}\left(\frac{k}{n+k}\right) \\ & =(-1)^{\frac{3-k-n}{4}}\left(\frac{k}{n+k}\right) \end{align}
Así, \begin{align} \frac{1+i}{\sqrt{2}}\epsilon_{n+k}^{-1}\left(\frac{2n}{n+k}\right) & =exp \left( i \pi \frac{1}{4}\right)exp \left( -i \pi \frac{1}{2}\right)exp \left( i \pi \frac{3-k-n}{4}\right)\left(\frac{k}{n+k}\right) \\ & =exp \left( i \pi \frac{2-k-n}{4}\right)\left(\frac{k}{n+k}\right) \end{align}
Así, para el caso en que $n$ incluso, tenemos $$\sum_{j=1}^{n}{(-1)^jexp \left( \frac{2 \pi i j^2k}{2n} \right)}=\sqrt{n}exp\left(i \pi \frac{2-k-n}{4}\right)\left( \frac{k}{n+k}\right)$$
Finalmente, combinando, tenemos
\begin{equation} \sum_{j=1}^{n}{(-1)^jexp \left( \frac{2 \pi i j^2k}{2n} \right)}= \begin{cases} -1 & \text{if $n$ is odd and $k$ is even} \\ \sqrt{n}exp\left(i \pi \frac{1-n}{4}\right)\left( \frac{k}{n}\right) & \text{if $n$ is odd and $k$ is odd} \\ \sqrt{n}exp\left(i \pi \frac{2-k-n}{4}\right)\left( \frac{k}{n+k}\right) & \text{if $n$ is even (so $k$ is odd)} \end{cases} \end{equation}
