\(\text{a) }\left(x^2+x\right)^2+4\left(x^2+x\right)=12\\ \Leftrightarrow\text{Đặt }x^2+x=y\\ \Leftrightarrow y^2+4y=12\\ \Leftrightarrow y^2+6y-2y-12=0\\ \Leftrightarrow\left(y^2+6y\right)-\left(2y+12\right)=0\\ \Leftrightarrow y\left(y+6\right)-2\left(y+6\right)=0\\ \Leftrightarrow\left(y+6\right)\left(y-2\right)=0\\ \Leftrightarrow\left(x^2+x+6\right)\left(x^2+x-2\right)=0\\ \Leftrightarrow\left(x^2+x+\dfrac{1}{4}+\dfrac{23}{4}\right)\left(x^2+2x-x-2\right)=0\\ \Leftrightarrow\left[\left(x^2+x+\dfrac{1}{4}\right)+\dfrac{23}{4}\right]\left[\left(x^2+2x\right)-\left(x+2\right)\right]=0\\ \Leftrightarrow\left[\left(x+\dfrac{1}{2}\right)^2+\dfrac{23}{4}\right]\left[x\left(x+2\right)-\left(x+2\right)\right]=0\\ \Leftrightarrow\left(x-1\right)\left(x+2\right)=0\left(Vì\text{ }\left(x+\dfrac{1}{2}\right)^2+\dfrac{23}{4}e0\right)\\ \Leftrightarrow\left[{}\begin{matrix}x-1=0\\x+2=0\end{matrix}\right.\Leftrightarrow\left[{}\begin{matrix}x=1\\x=-2\end{matrix}\right.\\ \text{Vậy }S=\left\{1;-2\right\}\\ \)

\(\text{b) }6x^4-5x^3-38x^2-5x+6=0\\ \Leftrightarrow x^2\left(6x^2-5x-38-\dfrac{5}{x}+\dfrac{6}{x^2}\right)=0\\ \Leftrightarrow x^2\left[\left(6x^2+12+\dfrac{6}{x^2}\right)-\left(5x+\dfrac{5}{x}\right)-50\right]=0\\ \Leftrightarrow x^2\left[6\left(x^2+2+\dfrac{1}{x^2}\right)-5\left(x+\dfrac{1}{x}\right)-50\right]=0\\ \Leftrightarrow x^2\left[6\left(x+\dfrac{1}{x}\right)^2-5\left(x+\dfrac{1}{x}\right)-50\right]=0\\ \text{Đặt }x+\dfrac{1}{x}=y\\ \Leftrightarrow x^2\left(6y^2-5y-50\right)=0\\ \Leftrightarrow x^2\left(6y^2-20y+15y-50\right)=0\\ \Leftrightarrow x^2\left[\left(6y^2-20y\right)+\left(15y-50\right)\right]=0\\ \Leftrightarrow x^2\left[2y\left(3y-10\right)+5\left(3y-10\right)\right]=0\\ \Leftrightarrow x^2\left(2y+5\right)\left(3y-10\right)=0\\ \Leftrightarrow x^2\left(2x+\dfrac{2}{x}+5\right)\left(3x+\dfrac{3}{x}-10\right)=0\\ \Leftrightarrow\left(2x^2+2+5x\right)\left(3x^2+3-10x\right)=0\\ \Leftrightarrow\left(2x^2+4x+x+2\right)\left(3x^2-9x-x+3\right)=0\\ \Leftrightarrow\left[\left(2x^2+4x\right)+\left(x+2\right)\right]\left[\left(3x^2-9x\right)-\left(x-3\right)\right]=0\\ \Leftrightarrow\left[2x\left(x+2\right)+\left(x+2\right)\right]\left[3x\left(x-3\right)-\left(x-3\right)\right]=0\\ \Leftrightarrow\left(2x+1\right)\left(x+2\right)\left(3x-1\right)\left(x-3\right)=0\\ \)

\(\Leftrightarrow\left[{}\begin{matrix}2x+1=0\\x+2=0\\3x-1=0\\x-3=0\end{matrix}\right.\Leftrightarrow\left[{}\begin{matrix}2x=-1\\x=-2\\3x=1\\x=3\end{matrix}\right.\Leftrightarrow\left[{}\begin{matrix}x=-\dfrac{1}{2}\\x=-2\\x=\dfrac{1}{3}\\x=3\end{matrix}\right.\\ \text{Vậy }S=\left\{-\dfrac{1}{2};-2;\dfrac{1}{3};3\right\}\)