$$I(t)=\int_0^{\infty} \frac{x}{x^2+b^2}\ln\left(\frac{x^2+2ax\cos(t)+a^2}{x^2-2ax\cos(t)+a^2}\right)\,dx$ $ $$I'(t)=\int_0^{\infty} \frac{x}{x^2+b^2}\left(\frac{-2ax\sin(t)}{x^2+2ax\cos(t)+a^2}-\frac{2ax\sin(t)}{x^2-2ax\cos(t)+a^2}\right)\,dx$ $ $$\begin{aligned}
\Rightarrow I'(t) \,& =-2a\int_0^{\infty}\frac{x}{x^2+b^2}\left(\frac{x\sin(t)}{x^2+2ax\cos(t)+a^2}+\frac{x\sin(t)}{x^2-2ax\cos(t)+a^2}\right)\,dx\\
& =-2a\,\Im\left(\int_0^{\infty} \frac{x}{x^2+b^2}\left(\frac{e^{it}}{x+ae^{it}}+\frac{e^{it}}{x-ae^{it}}\right)\,dx\right)\,\,\,\,\,\,\,\,(1)\\
& = -2a\,\Im\left(2e^{it} \int_0^{\infty} \frac{x^2}{(x^2+b^2)(x^2-a^2e^{2it})}\,dx \right)\\
& = -2a\,\Im\left(2e^{it}\frac{\pi}{2(b-iae^{it})}\right)\,\,\,\,\,\,\,\,(2)\\
& = -2\pi a\,\Im\left(\frac{e^{it}}{b-iae^{it}}\right)\\
& = -2\pi a\,\frac{b\sin t+a}{b^2+2ab\sin t+a^2}\,\,\,\,\,\,(3)\\
& = -\pi\left(1+\frac{a^2-b^2}{b^2+2ab\sin(t)+a^2}\right)\\
\end{alineados} $$
$$\Rightarrow I(t)=-\pi t-2\pi\arctan\left(\frac{(a^2+b^2)\tan(t/2)+2ab}{a^2-b^2}\right)+C\,\,\,\,\,\,(4)$$
Since $I(\pi/2)=0$, $C=\frac{\pi^2}{2}+2\pi\arctan\left(\frac{a+b}{a-b}\right)$.
%#% $ De #% la expresión anterior puede ser demostrada equivalente como se muestra en la OP mediante la simplificación arctangents: $$-$$ \begin{aligned}
-2\pi\left(\arctan\left(\frac{(a^2+b^2)\tan(t/2)+2ab}{a^2-b^2}\right)-\arctan\left(\frac{a+b}{a-b}\right)\right) &=2\pi\arctan\left(\frac{a-b}{a+b}\frac{1-\sin(t)}{\cos t}\right)\\
&=\pi\arctan\left(\frac{(a^2-b^2)\cos(t)}{(a^2+b^2)\sin(t)+2ab}\right)
\end{alineado} del ahí $$\Rightarrow I(t)=\frac{\pi^2}{2}-\pi t-2\pi\arctan\left(\frac{(a^2+b^2)\tan(t/2)+2ab}{a^2-b^2}\right)+2\pi\arctan\left(\frac{a+b}{a-b}\right)$ $
Prueba de $$I(t)=\frac{\pi^2}{2}-\pi t+\pi\arctan\left(\frac{(a^2-b^2)\cos(t)}{(a^2+b^2)\sin(t)+2ab}\right)$: $$\begin{aligned}
\frac{x\sin t}{x^2+2ax\cos(t)+a^2}&=\frac{1}{2i}\frac{x(e^{it}-e^{-it})}{x^2+axe^{it}+axe^{-it}+a^2}\\
&= \frac{1}{2i}\left(\frac{e^{it}}{x+ae^{it}}-\frac{e^{-it}}{x+ae^{-it}}\right)\\
&= \Im\left(\frac{e^{it}}{x+ae^{it}}\right)\,\,\,\,\,\,\left(\because \Im(z)=\frac{1}{2i}(z-\bar{z})\right)\\
\end{alineado} $$
De manera similar puede demostrarse que: $(1)$ $
Prueba $$\frac{x\sin t}{x^2-2ax\cos(t)+a^2}=\Im\left(\frac{e^{it}}{x-ae^{it}}\right)$: $$\begin{aligned}
\int_0^{\infty} \frac{x^2}{(x^2+b^2)(x^2-a^2e^{2it})}\,dx &=\frac{1}{b^2+a^2e^{2it}}\int_0^{\infty} \left(\frac{x^2}{x^2+b^2}-\frac{x^2}{x^2-a^2e^{2it}}\right)\,dx\\
&= -\frac{1}{b^2+a^2e^{2it}}\int_0^{\infty} \left(\frac{b^2}{x^2+b^2}+\frac{a^2e^{2it}}{x^2-a^2e^{2it}}\right)\,dt\\
&=\frac{1}{b^2+a^2e^{2it}} \left(\frac{\pi b}{2}+\frac{\pi iae^{it}}{2}\right)\\
&=\frac{\pi}{2}\frac{1}{(b+iae^{it})(b-ia^{it})}(b+iae^{it})\\
&=\frac{\pi}{2(b-iae^{it})}\\
\end{alineado} $$
Prueba $(2)$:
$$\begin{aligned}
\Im\left(\frac{e^{it}}{b-iae^{it}}\right) &= \Im\left(\frac{\cos(t)+i\sin(t)}{(b+a\sin(t))-ia\cos(t)}\right)\\
&=\Im\left(\frac{(cos(t)+i\sin(t))(b+a\sin(t)+ia\cos(t))}{(b+\sin(t))^2+a^2\cos^2(t)}\right)\\
&= \frac{b\sin (t)+a}{b^2+2ab\sin(t)+a^2}\\
\end{alineados} $$
Prueba $(3)$: $$\begin{aligned}
\int \frac{a^2-b^2}{b^2+2ab\sin(t)+a^2}\,dt &=(a^2-b^2)\int \frac{\sec^2(t/2)}{(a^2+b^2)(1+\tan^2(t/2))+4ab\tan(t/2)}\,dt\\
&= 2(a^2-b^2)\int \frac{dp}{(a^2+b^2)(1+p^2)+4abp}\,\,\,\,\,\,\,\,\,\left(p=\tan(t/2)\right)\\
&=2\frac{a^2-b^2}{a^2+b^2}\int\frac{dp}{p^2+\frac{4abp}{a^2+b^2}+1}\\
&=2\frac{a^2-b^2}{a^2+b^2}\int \frac{dp}{\left(p+\frac{2ab}{a^2+b^2}\right)^2+\left(\frac{a^2-b^2}{a^2+b^2}\right)^2}\\
&=2\arctan\left(\frac{(a^2+b^2)\tan(t/2)+2ab}{a^2-b^2}\right)+C\\
\end{alineado} $$