\[ y(x) \left (-a^2 \cos ^2(x)-\frac {b^2}{(2 a-3)^2}-3 a-b \cos (x)-2\right )+\sin ^2(x) y''(x)=0 \] ✓ Mathematica : cpu = 3.07327 (sec), leaf count = 4128
DSolve[(-2 - 3*a - b^2/(-3 + 2*a)^2 - b*Cos[x] - a^2*Cos[x]^2)*y[x] + Sin[x]^2*Derivative[2][y][x] == 0,y[x],x]
\[\left \{\left \{y(x)\to \frac {c_1 (\cos (x)+1)^{\frac {1}{2} \left (-\frac {8 a^2}{-16 a^2+48 a-36}+\frac {24 a}{-16 a^2+48 a-36}+a-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \left (-8 a+\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}+\sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}\right )+\frac {\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}}{4 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}-\frac {18}{-16 a^2+48 a-36}+1\right )} \operatorname {Hypergeometric2F1}\left (-\frac {8 a^2}{-16 a^2+48 a-36}+\frac {24 a}{-16 a^2+48 a-36}+a+\frac {\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}}{4 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}-\frac {18}{-16 a^2+48 a-36},\frac {1}{8} \left (-8 a+\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}+\sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}\right ),\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )},\frac {1}{2} (1-\cos (x))\right ) (\cos (x)-1)^{\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{4 \left (-16 a^2+48 a-36\right )}}}{\sqrt [4]{\cos ^2(x)-1}}+\frac {\left (-\frac {1}{2}\right )^{1-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}} c_2 (1-\cos (x))^{1-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}} (\cos (x)+1)^{\frac {1}{2} \left (-\frac {8 a^2}{-16 a^2+48 a-36}+\frac {24 a}{-16 a^2+48 a-36}+a-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \left (-8 a+\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}+\sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}\right )+\frac {\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}}{4 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}-\frac {18}{-16 a^2+48 a-36}+1\right )} \operatorname {Hypergeometric2F1}\left (-\frac {8 a^2}{-16 a^2+48 a-36}+\frac {24 a}{-16 a^2+48 a-36}+a-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}+\frac {\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}}{4 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}-\frac {18}{-16 a^2+48 a-36}+1,-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \left (-8 a+\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}+\sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}\right )+1,2-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )},\frac {1}{2} (1-\cos (x))\right ) (\cos (x)-1)^{\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{4 \left (-16 a^2+48 a-36\right )}}}{\sqrt [4]{\cos ^2(x)-1}}\right \}\right \}\] ✓ Maple : cpu = 0.743 (sec), leaf count = 558
dsolve(sin(x)^2*diff(diff(y(x),x),x)-(a^2*cos(x)^2+b*cos(x)+b^2/(2*a-3)^2+3*a+2)*y(x),y(x))
\[y \left (x \right ) = \frac {\left (\frac {\cos \left (x \right )}{2}-\frac {1}{2}\right )^{\frac {4 a -6+\sqrt {16 a^{4}+\left (16 b -72\right ) a^{2}-48 a b +4 \left (\frac {9}{2}+b \right )^{2}}}{8 a -12}} \left (\operatorname {hypergeom}\left (\left [\frac {8 a^{2}+\sqrt {16 a^{4}+\left (16 b -72\right ) a^{2}-48 a b +4 \left (\frac {9}{2}+b \right )^{2}}+\sqrt {16 a^{4}+\left (-16 b -72\right ) a^{2}+48 a b +4 \left (b -\frac {9}{2}\right )^{2}}-8 a -6}{8 a -12}, \frac {-8 a^{2}+\sqrt {16 a^{4}+\left (16 b -72\right ) a^{2}-48 a b +4 \left (\frac {9}{2}+b \right )^{2}}+\sqrt {16 a^{4}+\left (-16 b -72\right ) a^{2}+48 a b +4 \left (b -\frac {9}{2}\right )^{2}}+16 a -6}{8 a -12}\right ], \left [\frac {4 a -6+\sqrt {16 a^{4}+\left (-16 b -72\right ) a^{2}+48 a b +4 \left (b -\frac {9}{2}\right )^{2}}}{4 a -6}\right ], \frac {\cos \left (x \right )}{2}+\frac {1}{2}\right ) \cos \left (\frac {x}{2}\right )^{\frac {4 a -6+\sqrt {16 a^{4}+\left (-16 b -72\right ) a^{2}+48 a b +4 \left (b -\frac {9}{2}\right )^{2}}}{4 a -6}} c_{2}+\operatorname {hypergeom}\left (\left [\frac {8 a^{2}+\sqrt {16 a^{4}+\left (16 b -72\right ) a^{2}-48 a b +4 \left (\frac {9}{2}+b \right )^{2}}-\sqrt {16 a^{4}+\left (-16 b -72\right ) a^{2}+48 a b +4 \left (b -\frac {9}{2}\right )^{2}}-8 a -6}{8 a -12}, \frac {-8 a^{2}+\sqrt {16 a^{4}+\left (16 b -72\right ) a^{2}-48 a b +4 \left (\frac {9}{2}+b \right )^{2}}-\sqrt {16 a^{4}+\left (-16 b -72\right ) a^{2}+48 a b +4 \left (b -\frac {9}{2}\right )^{2}}+16 a -6}{8 a -12}\right ], \left [\frac {4 a -6-\sqrt {16 a^{4}+\left (-16 b -72\right ) a^{2}+48 a b +4 \left (b -\frac {9}{2}\right )^{2}}}{4 a -6}\right ], \frac {\cos \left (x \right )}{2}+\frac {1}{2}\right ) \cos \left (\frac {x}{2}\right )^{\frac {4 a -6-\sqrt {16 a^{4}+\left (-16 b -72\right ) a^{2}+48 a b +4 \left (b -\frac {9}{2}\right )^{2}}}{4 a -6}} c_{1}\right )}{\sqrt {\sin \left (x \right )}}\]