CREATE TABLE #tmp(SITE INT, id INT);
INSERT INTO #tmp VALUES 
(1,1),(1,2), (1,3), (1,4), (1,5),
(2,1),(2,2), (2,3), (2,4), (2,5),
(3,1),(3,2), (3,3), (3,4), 
(4,5)

DECLARE @sites INT = 1, @records INT = 5;

;WITH c1 AS
(
	SELECT
		*,
		ROW_NUMBER() OVER (PARTITION BY SITE ORDER BY NEWID())  AS RN,
		COUNT(*) OVER (PARTITION BY SITE) AS N
	FROM
		#tmp
),
c2 AS
( 
	SELECT TOP (@sites) SITE FROM c1 WHERE N >= @records GROUP BY SITE ORDER BY NEWID()
)
SELECT c1.* FROM c1 INNER JOIN c2 ON c1.site = c2.site
WHERE c1.RN <= @records;

DROP TABLE #tmp;