\[ y''(x)=y(x) \left (-\csc ^2(x)\right ) \left (-\left (\left (a^2 b^2-(a+1)^2\right ) \sin ^2(x)\right )-a (a+1) b \sin (2 x)+(1-a) a\right ) \] ✓ Mathematica : cpu = 0.715982 (sec), leaf count = 129
DSolve[Derivative[2][y][x] == -(Csc[x]^2*((1 - a)*a - (-(1 + a)^2 + a^2*b^2)*Sin[x]^2 - a*(1 + a)*b*Sin[2*x])*y[x]),y[x],x]
\[\left \{\left \{y(x)\to c_2 \left (e^{-a b x} \sin ^{-a-1}(x)+\frac {(2 a+1) \left (-1+e^{2 i x}\right ) e^{-a b x} \sin ^{a-2 (a+1)}(x) \, _2F_1\left (1,i a (b+i);i b a+a+2;e^{2 i x}\right ) (b \sin (x)+\cos (x))}{2 (a (b-i)-i)}\right )+c_1 e^{a b x} \sin ^a(x) (b \sin (x)+\cos (x))\right \}\right \}\] ✓ Maple : cpu = 1.224 (sec), leaf count = 203
dsolve(diff(diff(y(x),x),x) = -(-(a^2*b^2-(a+1)^2)*sin(x)^2-a*(a+1)*b*sin(2*x)-a*(a-1))/sin(x)^2*y(x),y(x))
\[y \left (x \right ) = \frac {{\mathrm e}^{\int \frac {\left (a \,b^{2}-a -2\right ) \left (\cos ^{2}\left (2 x \right )\right )+\left (-2 b \left (a +1\right ) \sin \left (2 x \right )-2 a -1\right ) \cos \left (2 x \right )+\left (-2 a -1\right ) b \sin \left (2 x \right )-a \,b^{2}-a +1}{\left (\cos \left (2 x \right )+1\right ) \left (\cos \left (2 x \right ) b -\sin \left (2 x \right )-b \right )}d x} \left (\left (\int -2 \,{\mathrm e}^{-2 \left (\int \frac {\left (a \,b^{2}-a -2\right ) \left (\cos ^{2}\left (2 x \right )\right )+\left (-2 b \left (a +1\right ) \sin \left (2 x \right )-2 a -1\right ) \cos \left (2 x \right )+\left (-2 a -1\right ) b \sin \left (2 x \right )-a \,b^{2}-a +1}{\left (\cos \left (2 x \right )+1\right ) \left (\cos \left (2 x \right ) b -\sin \left (2 x \right )-b \right )}d x \right )} \sin \left (2 x \right )d x \right ) c_{2}+c_{1}\right )}{\sqrt {\sin \left (2 x \right )}}\]