Custom Grid sorting function

I faced the same problem today. I solved it by adding joins to the _prepareCollection which I added to the filter_condition_callback. Then added the table.column name for the 'index' in $this->addColumn. (In my case: catalog_product_entity.sku).

In the _prepareCollection I added:

protected function _prepareCollection()
{
    $collection = Mage::getResourceModel($this->_getCollectionClass());

    // add to make skus sortable
    $collection->getSelect()->join(
                'sales_flat_order_item',
                'main_table.entity_id=sales_flat_order_item.order_id',
                array('product_id')
                )
            ->join(
                'catalog_product_entity',
                'sales_flat_order_item.product_id=catalog_product_entity.entity_id',
                array('sku')
            );

    $this->setCollection($collection);
    return parent::_prepareCollection();
}

This link helped me too: Column isn't Sorting in Custom Admin Report

for fix sort - need override method _setCollectionOrder,

for example my solution

protected function _setCollectionOrder($column)
{
    if (!$dir = $column->getDir()) {
        return $this;
    }

    if ($column->getIndex() == 'orders_count') {
        $collection = $this->getCollection();
        $collection->getSelect()
            ->order("orders_count " . strtoupper($column->getDir()));
        return $this;
    }

    if ($column->getIndex() == 'orders_total') {
        $collection = $this->getCollection();
        $collection->getSelect()
            ->order("orders_total " . strtoupper($column->getDir()));
        return $this;
    }

    return parent::_setCollectionOrder($column);
}