Try this:
SELECT custid
,NAME
,trandate
,appliedamt
FROM (
SELECT a.custid
,a.NAME
,b.trandate
,b.appliedamt
,rn = row_number() OVER (
ORDER BY b.trandate DESC
)
FROM customer a
LEFT JOIN cashhead b ON a.company = b.company
AND a.custid = b.custid
WHERE a.company = 'SF06'
AND b.trandate IN (
SELECT TOP 4 trandate
FROM cashhead u
WHERE u.company = b.company
AND u.custid = b.custid
AND u.trantype = 'PayInv'
)
)
WHERE rn <= 4