ClearAll[x]; integrand = Sin[x] ArcTan[Sqrt[Sec[x] - 1]]; res = Integrate[integrand, x] TeXForm[res]
\[ -\frac {1}{2} \left (-3-2 \sqrt {2}\right ) \left (\left (\sqrt {2}-2\right ) \cos \left (\frac {x}{2}\right )-\sqrt {2}+1\right ) \cos ^2\left (\frac {x}{4}\right ) \sqrt {-\tan ^2\left (\frac {x}{4}\right )-2 \sqrt {2}+3} \sqrt {\left (2 \sqrt {2}-3\right ) \tan ^2\left (\frac {x}{4}\right )+1} \cot \left (\frac {x}{4}\right ) \sqrt {\sec (x)-1} \sec (x) \sqrt {\left (\left (10-7 \sqrt {2}\right ) \cos \left (\frac {x}{2}\right )-5 \sqrt {2}+7\right ) \sec ^2\left (\frac {x}{4}\right )} \sqrt {\left (\left (2+\sqrt {2}\right ) \cos \left (\frac {x}{2}\right )-\sqrt {2}-1\right ) \sec ^2\left (\frac {x}{4}\right )} \left (\text {EllipticF}\left (\sin ^{-1}\left (\frac {\tan \left (\frac {x}{4}\right )}{\sqrt {3-2 \sqrt {2}}}\right ),17-12 \sqrt {2}\right )+2 \text {EllipticPi}\left (2 \sqrt {2}-3,-\sin ^{-1}\left (\frac {\tan \left (\frac {x}{4}\right )}{\sqrt {3-2 \sqrt {2}}}\right ),17-12 \sqrt {2}\right )\right )+\frac {1}{2} \cos (x) \sqrt {\sec (x)-1}-\cos (x) \tan ^{-1}\left (\sqrt {\sec (x)-1}\right ) \]
<< Rubi` ClearAll[x] integrand = Sin[x] ArcTan[Sqrt[Sec[x] - 1]]; res = Int[integrand, x]; TeXForm[res]
\[ \frac {1}{2} \cos (x) \sqrt {\sec (x)-1}+\frac {1}{2} \tan ^{-1}\left (\sqrt {\sec (x)-1}\right )-\cos (x) \tan ^{-1}\left (\sqrt {\sec (x)-1}\right ) \]
restart; integrand := sin(x)*arctan(sqrt(sec(x) - 1)); res:=int(integrand,x); latex(res)
\[ -{\frac {1}{\sec \left ( x \right ) }\arctan \left ( \sqrt {- \left ( -1+ \left ( \sec \left ( x \right ) \right ) ^{-1} \right ) \sec \left ( x \right ) } \right ) }+{\frac {1}{2\,\sec \left ( x \right ) }\sqrt {\sec \left ( x \right ) -1}}+{\frac {1}{2}\arctan \left ( \sqrt {\sec \left ( x \right ) -1} \right ) } \]
set output tex off setSimplifyDenomsFlag(true) integrand := sin(x)*atan(sqrt(sec(x) - 1)); res:=integrate(integrand,x); latex(res)
\[ {{{\left ( -{2 \ {\cos \left ( {x} \right )}}+1 \right )} \ {\arctan \left ( {{ \sqrt {{{\sec \left ( {x} \right )} -1}}}} \right )}}+{{\cos \left ( {x} \right )} \ {\sqrt {{{-{\cos \left ( {x} \right )}+1} \over {\cos \left ( {x} \right )}}} }}} \over 2 \]
integrand : sin(x)*atan(sqrt(sec(x) - 1)); res : integrate(integrand,x); tex(res);
\[ -\cos x\,\arctan \left ({{\sqrt {1-\cos x}}\over {\sqrt {\cos x}}} \right )+{{\arctan \left ({{\sqrt {1-\cos x}}\over {\sqrt {\cos x}}} \right )}\over {2}}+{{\sqrt {1-\cos x}}\over {\left (2-{{2\,\left (\cos x- 1\right )}\over {\cos x}}\right )\,\sqrt {\cos x}}} \]
integrand := sin(x)*atan(sqrt(sec(x) - 1)); res := integrate(integrand,x); Warning, integration of abs or sign assumes constant sign by intervals (correct if the argument is real): Check [abs(cos(x))] Discontinuities at zeroes of cos(x) were not checked latex(res)
Unable to conver to latex. Here is the raw output
expr:=(1/2*asin(2*cos(x)-1)/(sign(cos(x))^2-1)+sign(cos(x))^2* atan(1/2*(sign(cos(x))^2+(-2*sqrt(-cos(x)^2+cos(x))+1)/(-2*cos(x)+1) +(-2*sqrt(-cos(x)^2+cos(x))+1)*sign(cos(x))^2/(-2*cos(x)+1)-1)/sign(cos(x)))/ ((sign(cos(x))^2-1)*sign(cos(x))))*sign(cos(x)) -cos(x)*atan(sqrt(-cos(x)^2+cos(x))*sign(cos(x))/cos(x));
>python Python 3.7.3 (default, Mar 27 2019, 22:11:17) [GCC 7.3.0] :: Anaconda, Inc. on linux from sympy import * x = symbols('x') integrand = sin(x)*atan(sqrt(sec(x) - 1)); res = integrate(integrand,x); latex(res)
\[ \text {did not solve} \]
evalin(symengine,'int(sin(x)*arctan(sqrt(sec(x) - 1)),x)')
\[ -\arctan \left ( \sqrt { \left ( \cos \left ( x \right ) \right ) ^{-1}-1} \right ) \cos \left ( x \right ) -{\frac {\cos \left ( x \right ) }{3} \left ( {\frac {3}{2}\arcsin \left ( \sqrt {\cos \left ( x \right ) } \right ) \left ( \cos \left ( x \right ) \right ) ^{-{\frac {3}{2}}}}-{ \frac {3}{2\,\cos \left ( x \right ) }\sqrt {1-\cos \left ( x \right ) }} \right ) \sqrt {1-\cos \left ( x \right ) }{\frac {1}{\sqrt { \left ( \cos \left ( x \right ) \right ) ^{-1}-1}}}} \]