[Fizzlemizz]

This might be closer to what you'r looking for, it doesn't account for tapped or dead.

if UnitExists("target") then
	local r, g, b
	local playerFaction, targetFaction = UnitFactionGroup("player"), UnitFactionGroup("target")
	if targetFaction == "Neutral" then -- level less than 20 Panda or 55-58 DK
		-- r, g, b, = 1, 1, 1 -- whatever color works for you
	elseif targetFaction and playerFaction ~= targetFaction then -- enemy faction NPC or player
		r, g, b = 1, 0, 0 -- red
	elseif UnitIsPlayer("target") then -- friendly player
		local _, Class = UnitClass("target")
		local Color = RAID_CLASS_COLORS[Class] or {r = 1, g = 1, b = 1}
		r, g, b = Color.r, Color.g, Color.b
	else -- non "factioned" NPC
		r, g, b = UnitSelectionColor("target") -- your "reaction" to the target (neutral, friendly, die you sob)
	end
	self.bg:SetVertexColor(r, g, b)
else -- target out of range
	self.bg:SetVertexColor(0, 0, 0) -- or hide or...
end

Someone might have better or more definitive solution.