$\newcommand{\bbx}[1]{\,\bbox[15px,border:1px groove armada]{\displaystyle{#1}}\,}
\newcommand{\llaves}[1]{\left\lbrace\,{#1}\,\right\rbrace}
\newcommand{\bracks}[1]{\left\lbrack\,{#1}\,\right\rbrack}
\newcommand{\dd}{\mathrm{d}}
\newcommand{\ds}[1]{\displaystyle{#1}}
\newcommand{\expo}[1]{\,\mathrm{e}^{#1}\,}
\newcommand{\ic}{\mathrm{i}}
\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\mrm}[1]{\mathrm{#1}}
\newcommand{\pars}[1]{\left(\,{#1}\,\right)}
\newcommand{\partiald}[3][]{\frac{\partial^{#1} #2}{\parcial #3^{#1}}}
\newcommand{\raíz}[2][]{\,\sqrt[#1]{\,{#2}\,}\,}
\newcommand{\totald}[3][]{\frac{\mathrm{d}^{#1} #2}{\mathrm{d} #3^{#1}}}
\newcommand{\verts}[1]{\left\vert\,{#1}\,\right\vert}$
Con $\ds{a \in \mathbb{R}\setminus\braces{-1,1}}$:
\begin{align}
&\int_{0}^{2\pi}{x^{2}\sin\pars{x} \over 1 - 2a\cos\pars{x} + a^{2}}
\,\dd x =
\int_{-\pi}^{\pi}{\pars{x^{2} + 2\pi x + \pi^{2}}\bracks{-\sin\pars{x}} \over
1 + 2a\cos\pars{x} + a^{2}}\,\dd x
\\[5mm] = &\
-4\pi\int_{0}^{\pi}{x\sin\pars{x} \over
\pars{a + \expo{\ic x}}\pars{a + \expo{-\ic x}}}\,\dd x
\\[5mm] = &\
-4\pi\int_{0}^{\pi}x\sin\pars{x}\,
\pars{{1 \over a + \expo{-\ic x}} - {1 \over a + \expo{\ic x}}}
\,{1 \over \expo{\ic x} - \expo{-\ic x}}\,\dd x
\\[5mm] = &\
-4\pi\int_{0}^{\pi}x\sin\pars{x}
\pars{{1 \over a + \expo{\ic x}} - {1 \over a + \expo{-\ic x}}}
{1 \over 2\ic\sin\pars{x}}\,\dd x =
-4\pi\,\Im\int_{0}^{\pi}{x \over a + \expo{\ic x}}\,\dd x
\label{1}\tag{1}
\end{align}
$\ds{\Large a \to 0}.$
\begin{align}
\lim_{a \to 0}\Im\int_{0}^{\pi}{x \over a + \expo{\ic x}}\,\dd x & = \bbx{\pi}
\end{align}
$\ds{\Large a \in \pars{-1,1}\setminus\braces{0}}.$
\begin{align}
\Im\int_{0}^{\pi}{x \over a + \expo{\ic x}}\,\dd x & =
\Im\int_{0}^{\pi}{x\expo{-\ic x} \over 1 + a\expo{-\ic x}}\,\dd x =
\Im\sum_{n = 0}^{\infty}\pars{-a}^{n}\
\overbrace{\int_{0}^{\pi}x\expo{-\pars{n + 1}\ic x}\,\dd x}
^{\ds{1 + \pars{-1}^{n} + \pars{n + 1}\ic\pi \over \pars{n + 1}^{2}}}
\\[5mm] & =
\pi\sum_{n = 0}^{\infty}{\pars{-a}^{n} \over n + 1} =
-\,{\pi \over a}\sum_{n = 1}^{\infty}{\pars{-a}^{n} \over n} =
\bbx{\pi\,{\ln\pars{1 + a} \over a}}\label{2}\tag{2}
\end{align}
$\ds{\Large a \in \pars{-\infty,-1}\cup\pars{1,\infty}}.$
\begin{align}
\Im\int_{0}^{\pi}{x \over a + \expo{\ic x}}\,\dd x & =
{1 \over a}\,\Im\int_{0}^{\pi}{x \over 1 + \pars{1/a}\expo{\ic x}}\,\dd x =
{1 \over a}\,\Im\int_{0}^{\pi}
{x\expo{-\ic x} \over \pars{1/a} + \expo{-\ic x}}\,\dd x
\\[5mm] & =
-\,{1 \over a^{2}}\,\Im\int_{0}^{\pi}{x \over \pars{1/a} + \expo{-\ic x}}\,\dd x =
{1 \over a^{2}}\,\Im\int_{0}^{\pi}{x \over \pars{1/a} + \expo{\ic x}}\,\dd x
\\[5mm] & =
{1 \over a^{2}}\,\pi\,{\ln\pars{1 + 1/a} \over 1/a} =
\pi\,{\ln\pars{1 + 1/a} \over a}\label{3}\tag{3}
\end{align}
Aquí, he utilizado, el resultado anterior, \eqref{2}.
Con \eqref{1}, \eqref{2} y \eqref{3}:
$$
\left.\int_{0}^{2\pi}{x^{2}\sin\pars{x} \over 1 - 2a\cos\pars{x} + a^{2}}
\,\dd x\,\right\vert_{\ un\ \en\ \mathbb{R}\setminus\llaves{-1,1}} =
\left\{\begin{array}{lcl}
\ds{-4\pi^{2}} & \mbox{if} & \ds{a = 0}
\\[2mm]
\ds{-4\pi^{2}\,{\ln\pars{1 + a} \over a}} & \mbox{if} & \ds{-1 < a < 1}
\\[2mm]
\ds{-4\pi^{2}\,{\ln\pars{1 + 1/a} \over a}} & \mbox{if} & \ds{\verts{a} > 1}
\end{array}\right.
$$
La siguiente imagen es un gráfico de $\ds{a \in \mathbb{R}}$: