Как определить с помощью AppleScript, если что-то играет?

Вы забыли end ifкоманду.

on mySay(p, myMessage)
    tell application "QuickTime Player" to set isQTplaying to (playing of documents contains true)
    tell application "Music" to set isMusicPlaying to (player state is playing)
    tell application "VLC" to set isVLCplaying to playing
    --tell application "Brave" to set isiBravePlaying to (player state is playing)
    --tell application "FireFox" to set isiFireFoxPlaying to (player state is playing)
    --tell application "Safari" to set isiSafariPlaying to (player state is playing)
    if isQTplaying or isMusicPlaying or isVLCplaying or isiBravePlaying or isiFireFoxPlaying or isiSafariPlaying then
    else
        set volume output volume p
        say myMessage
        set volume output volume myVolumeBefore
    end if
end mySay

Я внес некоторые коррективы в ваш код и удалил то, что не смог скомпилировать на своей стороне.

Это что-то большее, чем то, что вы ищете?

on mySay(p, myMessage)
    tell application "QuickTime Player" to set isQTplaying to ¬
        ((documents whose playing is true) is not {}) as boolean
    tell application "Music" to set isMusicPlaying to (player state is playing)
    tell application "VLC" to set isVLCplaying to playing
    if isQTplaying or isMusicPlaying or isVLCplaying then
        set myVolumeBefore to output volume of (get volume settings)
        set volume output volume p
        say myMessage
        set volume output volume myVolumeBefore
    else
        return
    end if
end mySay