use File::Find::Rule;

my $dir = '/path/';

my @files = File::Find::Rule->file->in( $dir );
my @no_links = grep{ !-l $_ }@files;

unlink @no_links;

my @dirs = File::Find::Rule->dir->in( $dir );
rmdir $_ for @dirs;