getElementsByClassName can take a function as a last parameter. For the init and out methods you could do that to avoid looping through the button elements two or three times:
init: function() {
YAHOO.util.Dom.getElementsByClassName('rollover', 'div', function(el) {YAHOO.util.Event.on(el, 'mouseover', roll.over); YAHOO.util.Event.on(el, 'mouseout', roll.out);});
},
...
out: function() {
YAHOO.util.Dom.getElementsByClassName('btn', 'div', this, function(el) {YAHOO.util.Dom.removeClass(el, 'btn-over');});
}