ClearAll[x] integrand = Cos[x]^2/Sqrt[Cos[x]^4 + Cos[x]^2 + 1]; res = Integrate[integrand, x]; TeXForm[res]
\[ -\frac {2 i \cos ^2(x) \sqrt {1-\frac {2 i \tan ^2(x)}{\sqrt {3}-3 i}} \sqrt {1+\frac {2 i \tan ^2(x)}{\sqrt {3}+3 i}} \text {EllipticPi}\left (\frac {3}{2}+\frac {i \sqrt {3}}{2},i \sinh ^{-1}\left (\sqrt {-\frac {2 i}{\sqrt {3}-3 i}} \tan (x)\right ),\frac {-\sqrt {3}+3 i}{\sqrt {3}+3 i}\right )}{\sqrt {-\frac {i}{\sqrt {3}-3 i}} \sqrt {8 \cos (2 x)+\cos (4 x)+15}} \]
<< Rubi` ClearAll[x] integrand = Cos[x]^2/Sqrt[Cos[x]^4 + Cos[x]^2 + 1]; res = Int[integrand, x]; TeXForm[res]
\[ -\frac {\left (1+\sqrt {3}\right ) \cos ^2(x) \left (\tan ^2(x)+\sqrt {3}\right ) \sqrt {\frac {\tan ^4(x)+3 \tan ^2(x)+3}{\left (\tan ^2(x)+\sqrt {3}\right )^2}} \text {EllipticF}\left (2 \tan ^{-1}\left (\frac {\tan (x)}{\sqrt [4]{3}}\right ),\frac {1}{4} \left (2-\sqrt {3}\right )\right )}{4 \sqrt [4]{3} \sqrt {\cos ^4(x) \left (\tan ^4(x)+3 \tan ^2(x)+3\right )}}+\frac {\left (2+\sqrt {3}\right ) \cos ^2(x) \left (\tan ^2(x)+\sqrt {3}\right ) \sqrt {\frac {\tan ^4(x)+3 \tan ^2(x)+3}{\left (\tan ^2(x)+\sqrt {3}\right )^2}} \text {EllipticPi}\left (\frac {1}{6} \left (3-2 \sqrt {3}\right ),2 \tan ^{-1}\left (\frac {\tan (x)}{\sqrt [4]{3}}\right ),\frac {1}{4} \left (2-\sqrt {3}\right )\right )}{4 \sqrt [4]{3} \sqrt {\cos ^4(x) \left (\tan ^4(x)+3 \tan ^2(x)+3\right )}}+\frac {\cos ^2(x) \tan ^{-1}\left (\frac {\tan (x)}{\sqrt {\tan ^4(x)+3 \tan ^2(x)+3}}\right ) \sqrt {\tan ^4(x)+3 \tan ^2(x)+3}}{2 \sqrt {\cos ^4(x) \left (\tan ^4(x)+3 \tan ^2(x)+3\right )}} \]
restart; integrand:= cos(x)^2/sqrt(cos(x)^4 + cos(x)^2 + 1); res:=int(integrand,x); latex(res)
\[ -2\,{\frac {\sqrt { \left ( \left ( \cos \left ( 2\,x \right ) \right ) ^{2}+4\,\cos \left ( 2\,x \right ) +7 \right ) \left ( \sin \left ( 2\,x \right ) \right ) ^{2}} \left ( -3+i\sqrt {3} \right ) \left ( \cos \left ( 2\,x \right ) +1 \right ) ^{2}}{ \left ( -1+i\sqrt {3} \right ) \sqrt { \left ( \cos \left ( 2\,x \right ) -1 \right ) \left ( \cos \left ( 2\,x \right ) +1 \right ) \left ( \cos \left ( 2\,x \right ) +2+i\sqrt {3} \right ) \left ( i\sqrt {3}-\cos \left ( 2\,x \right ) -2 \right ) }\sin \left ( 2\,x \right ) \sqrt { \left ( \cos \left ( 2\,x \right ) \right ) ^{2}+4\,\cos \left ( 2\,x \right ) +7}}\sqrt {{\frac { \left ( -1+i\sqrt {3} \right ) \left ( \cos \left ( 2\,x \right ) -1 \right ) }{ \left ( -3+i\sqrt {3} \right ) \left ( \cos \left ( 2\,x \right ) +1 \right ) }}}\sqrt {{\frac {\cos \left ( 2\,x \right ) +2+i \sqrt {3}}{ \left ( i\sqrt {3}+3 \right ) \left ( \cos \left ( 2\,x \right ) +1 \right ) }}}\sqrt {{\frac {i\sqrt {3}-\cos \left ( 2\,x \right ) -2}{ \left ( -3+i\sqrt {3} \right ) \left ( \cos \left ( 2\,x \right ) +1 \right ) }}}{\it EllipticPi} \left ( \sqrt {{\frac { \left ( -1+i\sqrt {3} \right ) \left ( \cos \left ( 2\,x \right ) -1 \right ) }{ \left ( -3+i\sqrt {3} \right ) \left ( \cos \left ( 2\,x \right ) +1 \right ) }}},{\frac {-3+i\sqrt {3}}{-1+i\sqrt {3}}},\sqrt {{\frac { \left ( -3+i\sqrt {3} \right ) \left ( 1+i\sqrt {3} \right ) }{ \left ( i \sqrt {3}+3 \right ) \left ( -1+i\sqrt {3} \right ) }}} \right ) } \]
set output tex off setSimplifyDenomsFlag(true) integrand:= cos(x)^2/sqrt(cos(x)^4 + cos(x)^2 + 1); res:=integrate(integrand,x); latex(res)
\[ {\arctan \left ( {{{2 \ {{{\cos \left ( {x} \right )}} \sp {3}} \ {\sin \left ( {x} \right )} \ {\sqrt {{{{{\cos \left ( {x} \right )}} \sp {4}}+{{{\cos \left ( {x} \right )}} \sp {2}}+1}}}} \over {{2 \ {{{\cos \left ( {x} \right )}} \sp {6}}} -1}}} \right )} \over 6 \]
integrand : cos(x)^2/sqrt(cos(x)^4 + cos(x)^2 + 1); res : integrate(integrand,x); latex(res)
\[ \text {did not solve} \]
integrand := cos(x)^2/sqrt(cos(x)^4 + cos(x)^2 + 1); res := integrate(integrand,x); latex(res)
\[ \text {did not solve} \]
>python Python 3.7.3 (default, Mar 27 2019, 22:11:17) [GCC 7.3.0] :: Anaconda, Inc. on linux from sympy import * x = symbols('x') integrand = integrand = cos(x)**2/sqrt(cos(x)**4 + cos(x)**2 + 1); res = integrate(integrand,x); latex(res)
\[ \text {did not solve} \]
evalin(symengine,'int(cos(x)^2/sqrt(cos(x)^4 + cos(x)^2 + 1),x)')
\[ \text {did not solve} \]