Guess I can check if the current group ID corresponds with the group I am in.
<?php
// Let's say my group ID = 1
$exclude_admins_mods = (bp_get_group_id() == 1) ? 0 : 1;
if (bp_group_has_members(array('exclude_admins_mods'=> $exclude_admins_mods))) :
/* Member details */
endif;
?>