5.
a)$\begin{array}{l} A = \dfrac{{3 - 4\cos 2a + \cos 4a}}{{3 + 4\cos 2a + \cos 4a}}\\ A = \dfrac{{\left( {\cos 4a + 1} \right) - 4\cos 2a + 2}}{{\left( {\cos 4a + 1} \right) + 4\cos 2a + 2}} = \dfrac{{2{{\cos }^2}2a - 4\cos 2a + 2}}{{2{{\cos }^2}2a + 4\cos 2a + 1}} = \dfrac{{2{{\left( {\cos 2a - 1} \right)}^2}}}{{2{{\left( {\cos 2a + 1} \right)}^2}}}\\ A = \dfrac{{{{\left( {2{{\sin }^2}a} \right)}^2}}}{{{{\left( {2{{\cos }^2}a} \right)}^2}}} = {\tan ^4}a \end{array}$
b)
$\begin{array}{l} \dfrac{{A + B}}{2} = \dfrac{\pi }{2} - \dfrac{C}{2} \to \sin \dfrac{A}{2} = \cos \left( {\dfrac{{B + C}}{2}} \right)\\ VT = \dfrac{{\sin A + \sin B - \sin C}}{{\cos A + \cos B - \cos C + 1}} = \dfrac{{2\sin \dfrac{A}{2}\cos \dfrac{A}{2} + 2\cos \left( {\dfrac{{B + C}}{2}} \right)\sin \left( {\dfrac{{B - C}}{2}} \right)}}{{\left( {\cos A + 1} \right) + \cos B - \cos C}}\\ VT = \dfrac{{2\sin \dfrac{A}{2}\left( {\cos \dfrac{A}{2} + \sin \left( {\dfrac{{B - C}}{2}} \right)} \right)}}{{2{{\cos }^2}\dfrac{A}{2} - 2\sin \left( {\dfrac{{B + C}}{2}} \right)\sin \left( {\dfrac{{B - C}}{2}} \right)}}\\ VT = \dfrac{{2\sin \dfrac{A}{2}\left( {\cos \dfrac{A}{2} + \sin \left( {\dfrac{{B - C}}{2}} \right)} \right)}}{{2\cos \dfrac{A}{2}\left( {\cos \dfrac{A}{2} - \sin \left( {\dfrac{{B - C}}{2}} \right)} \right)}} = \tan \dfrac{A}{2}.\dfrac{{\sin \left( {\dfrac{{B + C}}{2}} \right) + \sin \left( {\dfrac{{B - C}}{2}} \right)}}{{\sin \left( {\dfrac{{B + C}}{2}} \right) - \sin \left( {\dfrac{{B - C}}{2}} \right)}}\\ VT = \tan \dfrac{A}{2}.\dfrac{{2\sin \dfrac{B}{2}\cos \dfrac{C}{2}}}{{2\cos \dfrac{B}{2}\sin \dfrac{C}{2}}} = \tan \dfrac{A}{2}\tan \dfrac{B}{2}\cot \dfrac{C}{2} \end{array}$